mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-02-07 15:03:34 +00:00
fa20edc44d
- Remove most __ASSEMBLER__ __LINKER__ ifdefs - Rename libc/intrin/bits.h to libc/serialize.h - Block pthread cancelation in fchmodat() polyfill - Remove `clang-format off` statements in third_party
300 lines
11 KiB
C
300 lines
11 KiB
C
#ifndef _CJKCODECS_H_
|
|
#define _CJKCODECS_H_
|
|
#define PY_SSIZE_T_CLEAN
|
|
#include "third_party/python/Include/abstract.h"
|
|
#include "third_party/python/Include/import.h"
|
|
#include "third_party/python/Include/methodobject.h"
|
|
#include "third_party/python/Include/modsupport.h"
|
|
#include "third_party/python/Include/pycapsule.h"
|
|
#include "third_party/python/Include/pyerrors.h"
|
|
#include "third_party/python/Include/pymacro.h"
|
|
#include "third_party/python/Modules/cjkcodecs/multibytecodec.h"
|
|
#include "third_party/python/Modules/cjkcodecs/somanyencodings.h"
|
|
|
|
/* a unicode "undefined" code point */
|
|
#define UNIINV 0xFFFE
|
|
|
|
/* internal-use DBCS code points which aren't used by any charsets */
|
|
#define NOCHAR 0xFFFF
|
|
#define MULTIC 0xFFFE
|
|
#define DBCINV 0xFFFD
|
|
|
|
/* shorter macros to save source size of mapping tables */
|
|
#define U UNIINV
|
|
#define N NOCHAR
|
|
#define M MULTIC
|
|
#define D DBCINV
|
|
|
|
static const MultibyteCodec *codec_list;
|
|
|
|
#define CODEC_INIT(encoding) \
|
|
static int encoding##_codec_init(const void *config)
|
|
|
|
#define ENCODER_INIT(encoding) \
|
|
static int encoding##_encode_init( \
|
|
MultibyteCodec_State *state, const void *config)
|
|
#define ENCODER(encoding) \
|
|
static Py_ssize_t encoding##_encode( \
|
|
MultibyteCodec_State *state, const void *config, \
|
|
int kind, void *data, \
|
|
Py_ssize_t *inpos, Py_ssize_t inlen, \
|
|
unsigned char **outbuf, Py_ssize_t outleft, int flags)
|
|
#define ENCODER_RESET(encoding) \
|
|
static Py_ssize_t encoding##_encode_reset( \
|
|
MultibyteCodec_State *state, const void *config, \
|
|
unsigned char **outbuf, Py_ssize_t outleft)
|
|
|
|
#define DECODER_INIT(encoding) \
|
|
static int encoding##_decode_init( \
|
|
MultibyteCodec_State *state, const void *config)
|
|
#define DECODER(encoding) \
|
|
static Py_ssize_t encoding##_decode( \
|
|
MultibyteCodec_State *state, const void *config, \
|
|
const unsigned char **inbuf, Py_ssize_t inleft, \
|
|
_PyUnicodeWriter *writer)
|
|
#define DECODER_RESET(encoding) \
|
|
static Py_ssize_t encoding##_decode_reset( \
|
|
MultibyteCodec_State *state, const void *config)
|
|
|
|
#define NEXT_IN(i) \
|
|
do { \
|
|
(*inbuf) += (i); \
|
|
(inleft) -= (i); \
|
|
} while (0)
|
|
#define NEXT_INCHAR(i) \
|
|
do { \
|
|
(*inpos) += (i); \
|
|
} while (0)
|
|
#define NEXT_OUT(o) \
|
|
do { \
|
|
(*outbuf) += (o); \
|
|
(outleft) -= (o); \
|
|
} while (0)
|
|
#define NEXT(i, o) \
|
|
do { \
|
|
NEXT_INCHAR(i); \
|
|
NEXT_OUT(o); \
|
|
} while (0)
|
|
|
|
#define REQUIRE_INBUF(n) \
|
|
do { \
|
|
if (inleft < (n)) \
|
|
return MBERR_TOOFEW; \
|
|
} while (0)
|
|
|
|
#define REQUIRE_OUTBUF(n) \
|
|
do { \
|
|
if (outleft < (n)) \
|
|
return MBERR_TOOSMALL; \
|
|
} while (0)
|
|
|
|
#define INBYTE1 ((*inbuf)[0])
|
|
#define INBYTE2 ((*inbuf)[1])
|
|
#define INBYTE3 ((*inbuf)[2])
|
|
#define INBYTE4 ((*inbuf)[3])
|
|
|
|
#define INCHAR1 (PyUnicode_READ(kind, data, *inpos))
|
|
#define INCHAR2 (PyUnicode_READ(kind, data, *inpos + 1))
|
|
|
|
#define OUTCHAR(c) \
|
|
do { \
|
|
if (_PyUnicodeWriter_WriteChar(writer, (c)) < 0) \
|
|
return MBERR_EXCEPTION; \
|
|
} while (0)
|
|
|
|
#define OUTCHAR2(c1, c2) \
|
|
do { \
|
|
Py_UCS4 _c1 = (c1); \
|
|
Py_UCS4 _c2 = (c2); \
|
|
if (_PyUnicodeWriter_Prepare(writer, 2, Py_MAX(_c1, c2)) < 0) \
|
|
return MBERR_EXCEPTION; \
|
|
PyUnicode_WRITE(writer->kind, writer->data, writer->pos, _c1); \
|
|
PyUnicode_WRITE(writer->kind, writer->data, writer->pos + 1, _c2); \
|
|
writer->pos += 2; \
|
|
} while (0)
|
|
|
|
#define OUTBYTE1(c) \
|
|
do { ((*outbuf)[0]) = (c); } while (0)
|
|
#define OUTBYTE2(c) \
|
|
do { ((*outbuf)[1]) = (c); } while (0)
|
|
#define OUTBYTE3(c) \
|
|
do { ((*outbuf)[2]) = (c); } while (0)
|
|
#define OUTBYTE4(c) \
|
|
do { ((*outbuf)[3]) = (c); } while (0)
|
|
|
|
#define WRITEBYTE1(c1) \
|
|
do { \
|
|
REQUIRE_OUTBUF(1); \
|
|
(*outbuf)[0] = (c1); \
|
|
} while (0)
|
|
#define WRITEBYTE2(c1, c2) \
|
|
do { \
|
|
REQUIRE_OUTBUF(2); \
|
|
(*outbuf)[0] = (c1); \
|
|
(*outbuf)[1] = (c2); \
|
|
} while (0)
|
|
#define WRITEBYTE3(c1, c2, c3) \
|
|
do { \
|
|
REQUIRE_OUTBUF(3); \
|
|
(*outbuf)[0] = (c1); \
|
|
(*outbuf)[1] = (c2); \
|
|
(*outbuf)[2] = (c3); \
|
|
} while (0)
|
|
#define WRITEBYTE4(c1, c2, c3, c4) \
|
|
do { \
|
|
REQUIRE_OUTBUF(4); \
|
|
(*outbuf)[0] = (c1); \
|
|
(*outbuf)[1] = (c2); \
|
|
(*outbuf)[2] = (c3); \
|
|
(*outbuf)[3] = (c4); \
|
|
} while (0)
|
|
|
|
#define _TRYMAP_ENC(__m, m, assi, val) \
|
|
(m.map && (val) >= m.bottom && (val)<= m.top && \
|
|
((assi) = (__m() + m.map - 1)[(val) - m.bottom]) != NOCHAR)
|
|
#define TRYMAP_ENC(M, assi, uni) \
|
|
_TRYMAP_ENC(__##M##_encmap, M##_encmap()[(uni) >> 8], assi, (uni) & 0xff)
|
|
|
|
#define _TRYMAP_DEC(__m, m, assi, val) \
|
|
(m.map && (val) >= m.bottom && (val) <= m.top && \
|
|
((assi) = (__m() + m.map - 1)[(val) - m.bottom]) != UNIINV)
|
|
#define TRYMAP_DEC(M, assi, c1, c2) \
|
|
_TRYMAP_DEC(__##M##_decmap, M##_decmap()[c1], assi, c2)
|
|
|
|
#define BEGIN_MAPPINGS_LIST
|
|
#define MAPPING_ENCONLY(enc)
|
|
#define MAPPING_DECONLY(enc)
|
|
#define MAPPING_ENCDEC(enc)
|
|
#define END_MAPPINGS_LIST
|
|
|
|
#define BEGIN_CODECS_LIST static const MultibyteCodec _codec_list[] = {
|
|
#define _STATEFUL_METHODS(enc) \
|
|
enc##_encode, \
|
|
enc##_encode_init, \
|
|
enc##_encode_reset, \
|
|
enc##_decode, \
|
|
enc##_decode_init, \
|
|
enc##_decode_reset,
|
|
#define _STATELESS_METHODS(enc) \
|
|
enc##_encode, NULL, NULL, \
|
|
enc##_decode, NULL, NULL,
|
|
#define CODEC_STATEFUL(enc) { \
|
|
#enc, NULL, NULL, \
|
|
_STATEFUL_METHODS(enc) \
|
|
},
|
|
#define CODEC_STATELESS(enc) { \
|
|
#enc, NULL, NULL, \
|
|
_STATELESS_METHODS(enc) \
|
|
},
|
|
#define CODEC_STATELESS_WINIT(enc) { \
|
|
#enc, NULL, \
|
|
enc##_codec_init, \
|
|
_STATELESS_METHODS(enc) \
|
|
},
|
|
#define END_CODECS_LIST \
|
|
{"", NULL,} }; \
|
|
static const MultibyteCodec *codec_list = \
|
|
(const MultibyteCodec *)_codec_list;
|
|
|
|
static PyObject *
|
|
getmultibytecodec(void)
|
|
{
|
|
static PyObject *cofunc = NULL;
|
|
if (cofunc == NULL) {
|
|
PyObject *mod = PyImport_ImportModuleNoBlock("_multibytecodec");
|
|
if (mod == NULL)
|
|
return NULL;
|
|
cofunc = PyObject_GetAttrString(mod, "__create_codec");
|
|
Py_DECREF(mod);
|
|
}
|
|
return cofunc;
|
|
}
|
|
|
|
static PyObject *
|
|
getcodec(PyObject *self, PyObject *encoding)
|
|
{
|
|
PyObject *codecobj, *r, *cofunc;
|
|
const MultibyteCodec *codec;
|
|
const char *enc;
|
|
if (!PyUnicode_Check(encoding)) {
|
|
PyErr_SetString(PyExc_TypeError,
|
|
"encoding name must be a string.");
|
|
return NULL;
|
|
}
|
|
enc = PyUnicode_AsUTF8(encoding);
|
|
if (enc == NULL)
|
|
return NULL;
|
|
cofunc = getmultibytecodec();
|
|
if (cofunc == NULL)
|
|
return NULL;
|
|
for (codec = codec_list; codec->encoding[0]; codec++)
|
|
if (strcmp(codec->encoding, enc) == 0)
|
|
break;
|
|
if (codec->encoding[0] == '\0') {
|
|
PyErr_SetString(PyExc_LookupError,
|
|
"no such codec is supported.");
|
|
return NULL;
|
|
}
|
|
codecobj = PyCapsule_New((void *)codec, PyMultibyteCodec_CAPSULE_NAME, NULL);
|
|
if (codecobj == NULL)
|
|
return NULL;
|
|
r = PyObject_CallFunctionObjArgs(cofunc, codecobj, NULL);
|
|
Py_DECREF(codecobj);
|
|
return r;
|
|
}
|
|
|
|
#pragma GCC push_options
|
|
#pragma GCC diagnostic ignored "-Wunused-variable"
|
|
static struct PyMethodDef __methods[] = {
|
|
{"getcodec", (PyCFunction)getcodec, METH_O, ""},
|
|
{0},
|
|
};
|
|
#pragma GCC pop_options
|
|
|
|
#ifdef USING_BINARY_PAIR_SEARCH
|
|
static DBCHAR
|
|
find_pairencmap(ucs2_t body, ucs2_t modifier,
|
|
const struct CjkPairEncodeMap *haystack,
|
|
int haystacksize)
|
|
{
|
|
int pos, min, max;
|
|
Py_UCS4 value = body << 16 | modifier;
|
|
min = 0;
|
|
max = haystacksize;
|
|
for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) {
|
|
if (value < haystack[pos].uniseq) {
|
|
if (max != pos) {
|
|
max = pos;
|
|
continue;
|
|
}
|
|
}
|
|
else if (value > haystack[pos].uniseq) {
|
|
if (min != pos) {
|
|
min = pos;
|
|
continue;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
if (value == haystack[pos].uniseq) {
|
|
return haystack[pos].code;
|
|
}
|
|
return DBCINV;
|
|
}
|
|
#endif
|
|
|
|
#define I_AM_A_MODULE_FOR(loc) \
|
|
static struct PyModuleDef __module = { \
|
|
PyModuleDef_HEAD_INIT, \
|
|
"_codecs_"#loc, \
|
|
NULL, \
|
|
0, \
|
|
__methods, \
|
|
}; \
|
|
PyMODINIT_FUNC \
|
|
PyInit__codecs_##loc(void) \
|
|
{ \
|
|
return PyModule_Create(&__module); \
|
|
}
|
|
|
|
#endif
|