python/mysqlclient
1
0
Fork 0
mirror of https://github.com/PyMySQL/mysqlclient.git synced 2026-03-13 08:00:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Faster string and int decoder (#305)

Browse Source
This commit is contained in:
INADA Naoki
2018-12-11 11:50:42 +09:00
committed by GitHub
parent c7188e64bb
commit 3bea8cd8dd
3 changed files with 71 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
MySQLdb/_mysql.c
View File

@@ -37,6 +37,8 @@ PERFORMANCE OF THIS SOFTWARE.
#include "Python.h"
#if PY_MAJOR_VERSION >= 3
#define IS_PY3K
#define PyInt_Type PyLong_Type
#define PyInt_FromString PyLong_FromString
#define PyInt_FromLong(n) PyLong_FromLong(n)
#define PyInt_Check(n) PyLong_Check(n)
#define PyInt_AS_LONG(n) PyLong_AS_LONG(n)
@@ -87,6 +89,7 @@ typedef struct {
int use;
char has_next;
PyObject *converter;
const char *encoding;
} _mysql_ResultObject;
extern PyTypeObject _mysql_ResultObject_Type;
@@ -203,6 +206,25 @@ _mysql_Exception(_mysql_ConnectionObject *c)
return NULL;
}
static const char *utf8 = "utf8";
static const char*
_get_encoding(MYSQL *mysql)
{
MY_CHARSET_INFO cs;
mysql_get_character_set_info(mysql, &cs);
if (strncmp(utf8, cs.csname, 4) == 0) { // utf8, utf8mb3, utf8mb4
return utf8;
}
else if (strncmp("koi8r", cs.csname, 5) == 0) {
return "koi8_r";
}
else if (strncmp("koi8u", cs.csname, 5) == 0) {
return "koi8_u";
}
return cs.csname;
}
static char _mysql_thread_safe__doc__[] =
"Indicates whether the client is compiled as thread-safe.";
@@ -251,6 +273,9 @@ _mysql_ResultObject_Initialize(
self->result = result;
self->has_next = (char)mysql_more_results(&(conn->connection));
Py_END_ALLOW_THREADS ;
self->encoding = _get_encoding(&(conn->connection));
//fprintf(stderr, "encoding=%s\n", self->encoding);
if (!result) {
if (mysql_errno(&(conn->connection))) {
_mysql_Exception(conn);
@@ -289,6 +314,8 @@ _mysql_ResultObject_Initialize(
int j, n2=PySequence_Size(fun);
if (fields[i].charsetnr != 63) { /* maaagic */
flags &= ~BINARY_FLAG;
} else {
flags |= BINARY_FLAG;
}
for (j=0; j<n2; j++) {
PyObject *t = PySequence_GetItem(fun, j);
@@ -943,8 +970,6 @@ _mysql_string_literal(
return (str);
}
static PyObject *_mysql_NULL;
static PyObject *
_escape_item(
PyObject *self,
@@ -1011,17 +1036,10 @@ _mysql_ResultObject_describe(
{
PyObject *d;
MYSQL_FIELD *fields;
MY_CHARSET_INFO cs;
int isutf8 = 0;
unsigned int i, n;
check_result_connection(self);
mysql_get_character_set_info(&result_connection(self)->connection, &cs);
if (strncmp("utf8", cs.name, 4) == 0) { // utf8, utf8mb3, utf8mb4
isutf8 = 1;
}
n = mysql_num_fields(self->result);
fields = mysql_fetch_fields(self->result);
if (!(d = PyTuple_New(n))) return NULL;
@@ -1029,10 +1047,10 @@ _mysql_ResultObject_describe(
PyObject *t;
#ifdef IS_PY3K
PyObject *name;
if (isutf8) {
if (self->encoding == utf8) {
name = PyUnicode_DecodeUTF8(fields[i].name, fields[i].name_length, "replace");
} else {
name = PyUnicode_Decode(fields[i].name, fields[i].name_length, cs.name, "replace");
name = PyUnicode_Decode(fields[i].name, fields[i].name_length, self->encoding, "replace");
}
if (name == NULL) {
goto error;
@@ -1089,9 +1107,10 @@ _mysql_ResultObject_field_flags(
static PyObject *
_mysql_field_to_python(
PyObject *converter,
char *rowitem,
const char *rowitem,
unsigned long length,
MYSQL_FIELD *field)
MYSQL_FIELD *field,
const char *encoding)
{
PyObject *v;
#ifdef IS_PY3K
@@ -1110,7 +1129,27 @@ _mysql_field_to_python(
}
#endif
if (rowitem) {
if (converter != Py_None) {
if (converter == (PyObject*)&PyUnicode_Type) {
if (encoding == utf8) {
//fprintf(stderr, "decoding with utf8!\n");
v = PyUnicode_DecodeUTF8(rowitem, length, NULL);
} else {
//fprintf(stderr, "decoding with %s\n", encoding);
v = PyUnicode_Decode(rowitem, length, encoding, NULL);
}
}
else if (converter == (PyObject*)&PyBytes_Type) {
//fprintf(stderr, "decoding with bytes\n", encoding);
v = PyBytes_FromStringAndSize(rowitem, length);
}
else if (converter == (PyObject*)&PyInt_Type) {
//fprintf(stderr, "decoding with int\n", encoding);
v = PyInt_FromString(rowitem, NULL, 10);
}
else if (converter != Py_None) {
//fprintf(stderr, "decoding with callback\n");
//PyObject_Print(converter, stderr, 0);
//fprintf(stderr, "\n");
v = PyObject_CallFunction(converter,
#ifdef IS_PY3K
binary ? "y#" : "s#",
@@ -1120,6 +1159,7 @@ _mysql_field_to_python(
rowitem,
(int)length);
} else {
//fprintf(stderr, "converter=None\n");
#ifdef IS_PY3K
if (!binary) {
v = PyUnicode_FromStringAndSize(rowitem, (int)length);
@@ -1153,7 +1193,7 @@ _mysql_row_to_tuple(
for (i=0; i<n; i++) {
PyObject *v;
c = PyTuple_GET_ITEM(self->converter, i);
v = _mysql_field_to_python(c, row[i], length[i], &fields[i]);
v = _mysql_field_to_python(c, row[i], length[i], &fields[i], self->encoding);
if (!v) goto error;
PyTuple_SET_ITEM(r, i, v);
}
@@ -1180,7 +1220,7 @@ _mysql_row_to_dict(
for (i=0; i<n; i++) {
PyObject *v;
c = PyTuple_GET_ITEM(self->converter, i);
v = _mysql_field_to_python(c, row[i], length[i], &fields[i]);
v = _mysql_field_to_python(c, row[i], length[i], &fields[i], self->encoding);
if (!v) goto error;
if (!PyMapping_HasKeyString(r, fields[i].name)) {
PyMapping_SetItemString(r, fields[i].name, v);
@@ -1219,7 +1259,7 @@ _mysql_row_to_dict_old(
for (i=0; i<n; i++) {
PyObject *v;
c = PyTuple_GET_ITEM(self->converter, i);
v = _mysql_field_to_python(c, row[i], length[i], &fields[i]);
v = _mysql_field_to_python(c, row[i], length[i], &fields[i], self->encoding);
if (!v) goto error;
{
int len=0;
@@ -1422,8 +1462,7 @@ _mysql_ConnectionObject_set_character_set(
err = mysql_set_character_set(&(self->connection), s);
Py_END_ALLOW_THREADS
if (err) return _mysql_Exception(self);
Py_INCREF(Py_None);
return Py_None;
Py_RETURN_NONE;
}
#if MYSQL_VERSION_ID >= 50010
@@ -2634,12 +2673,10 @@ init_mysql(void)
_mysql_ConnectionObject_Type.ob_type = &PyType_Type;
_mysql_ResultObject_Type.ob_type = &PyType_Type;
#endif
#if PY_VERSION_HEX >= 0x02020000
_mysql_ConnectionObject_Type.tp_alloc = PyType_GenericAlloc;
_mysql_ConnectionObject_Type.tp_new = PyType_GenericNew;
_mysql_ResultObject_Type.tp_alloc = PyType_GenericAlloc;
_mysql_ResultObject_Type.tp_new = PyType_GenericNew;
#endif
#ifdef IS_PY3K
if (PyType_Ready(&_mysql_ConnectionObject_Type) < 0)
return NULL;

7
MySQLdb/connections.py
View File

@@ -114,7 +114,7 @@ class Connection(_mysql.connection):
documentation for the MySQL C API for some hints on what they do.
"""
from MySQLdb.constants import CLIENT, FIELD_TYPE
from MySQLdb.converters import conversions
from MySQLdb.converters import conversions, _bytes_or_str
from weakref import proxy
kwargs2 = kwargs.copy()
@@ -174,9 +174,6 @@ class Connection(_mysql.connection):
def unicode_literal(u, dummy=None):
return db.string_literal(u.encode(db.encoding))
def string_decoder(s):
return s.decode(db.encoding)
if not charset:
charset = self.character_set_name()
self.set_character_set(charset)
@@ -187,7 +184,7 @@ class Connection(_mysql.connection):
if use_unicode:
for t in (FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING, FIELD_TYPE.VARCHAR, FIELD_TYPE.TINY_BLOB,
FIELD_TYPE.MEDIUM_BLOB, FIELD_TYPE.LONG_BLOB, FIELD_TYPE.BLOB):
self.converter[t].append((None, string_decoder))
self.converter[t] = _bytes_or_str
self.encoders[unicode] = unicode_literal
self._transactional = self.server_capabilities & CLIENT.TRANSACTIONS

22
MySQLdb/converters.py
View File

@@ -35,7 +35,7 @@ from decimal import Decimal
from MySQLdb._mysql import string_literal, escape
from MySQLdb.constants import FIELD_TYPE, FLAG
from MySQLdb.times import *
from MySQLdb.compat import PY2, long
from MySQLdb.compat import PY2, long, unicode
NoneType = type(None)
@@ -91,7 +91,7 @@ def quote_tuple(t, d):
return "(%s)" % (','.join(escape_sequence(t, d)))
# bytes or str regarding to BINARY_FLAG.
_bytes_or_str = [(FLAG.BINARY, bytes)]
_bytes_or_str = ((FLAG.BINARY, bytes), (None, unicode))
conversions = {
int: Thing2Str,
@@ -109,12 +109,12 @@ conversions = {
FIELD_TYPE.TINY: int,
FIELD_TYPE.SHORT: int,
FIELD_TYPE.LONG: long,
FIELD_TYPE.LONG: int,
FIELD_TYPE.FLOAT: float,
FIELD_TYPE.DOUBLE: float,
FIELD_TYPE.DECIMAL: Decimal,
FIELD_TYPE.NEWDECIMAL: Decimal,
FIELD_TYPE.LONGLONG: long,
FIELD_TYPE.LONGLONG: int,
FIELD_TYPE.INT24: int,
FIELD_TYPE.YEAR: int,
FIELD_TYPE.TIMESTAMP: mysql_timestamp_converter,
@@ -122,11 +122,11 @@ conversions = {
FIELD_TYPE.TIME: TimeDelta_or_None,
FIELD_TYPE.DATE: Date_or_None,
FIELD_TYPE.TINY_BLOB: _bytes_or_str,
FIELD_TYPE.MEDIUM_BLOB: _bytes_or_str,
FIELD_TYPE.LONG_BLOB: _bytes_or_str,
FIELD_TYPE.BLOB: _bytes_or_str,
FIELD_TYPE.STRING: _bytes_or_str,
FIELD_TYPE.VAR_STRING: _bytes_or_str,
FIELD_TYPE.VARCHAR: _bytes_or_str,
FIELD_TYPE.TINY_BLOB: bytes,
FIELD_TYPE.MEDIUM_BLOB: bytes,
FIELD_TYPE.LONG_BLOB: bytes,
FIELD_TYPE.BLOB: bytes,
FIELD_TYPE.STRING: bytes,
FIELD_TYPE.VAR_STRING: bytes,
FIELD_TYPE.VARCHAR: bytes,
}