PY08-05:Python的C扩展模块

2020-05-16  本文已影响0人  杨强AT南京

  Python在提供傻白甜这样的编程方式的时候，对性能就是硬伤了，所以性能这块提供了C的扩展模块，实际上Python是与本地二进制可执行动态库是无缝调用的。上一个主题解释了Python调用动态库，但是那一种方式需要太多C的知识，其实Python还提供了一种无缝自动的调用方式，在Python成为built-in实现。这个主题就介绍Python的C扩展。顺便还介绍了怎么制作Python安装包

Python的C扩展的模式

Python与C扩展动态库之间的工作关系

Python与C扩展动态库的工作关系

C动态库的编译

C扩展接口的编程模式

包装函数实现


static char py_gcd_doc[] = "最大公约数";
static PyObject* py_gcd(PyObject*self, PyObject *args){
    int r, x, y;
    // 解析参数
    if(!PyArg_ParseTuple(args, "ii:gcd", &x, &y)){
        return NULL;
    }
    r = gcd(x, y);
    return Py_BuildValue("i", r); 
}

static char py_replace_doc[] = "字符替换";
static PyObject* py_replace(PyObject* self, PyObject *args, PyObject *kwargs){
    char *s, *sdup;
    char och, nch;
    int nrep;

    PyObject *result;
    static char *argsname[] = {"s", "och", "nch", NULL};
    // 解析参数
    if(!PyArg_ParseTupleAndKeywords(args, kwargs, "scc:replace", argsname, &s, &och, &nch)){
        return NULL;
    }
    sdup = (char *)malloc(strlen(s) + 1);
    strcpy(sdup, s);
    nrep = replace(sdup, och, nch);
    result = Py_BuildValue("(is)", nrep, sdup);
    free(sdup);
    return result;
}

static char py_distance_doc[] = "计算距离";
static PyObject* py_distance(PyObject *self, PyObject *args){
    PyErr_SetString(PyExc_NotImplementedError, "distance() not implements");
    return NULL;
}

附录：上面调用的三个函数的C实现

头文件：C.h

#ifndef C_YQ_H
#define C_YQ_H
#include <stdio.h>
#include <string.h>
#include <math.h>

typedef struct Point{
    double x;
    double y;
} Point;


extern int gcd(int x, int y);
extern int replace(char *s, char och, char nch);
extern double distance(Point *a, Point *b);

#define MAGIC  0x31337

#endif

实现文件：C.c

#include "C.h"
int gcd(int x, int y){
    /* 
        这个算法是来自最大公约数的一个性质
        gcd(a,b)=gcd(b, a mod b)
        gcd(a,b)=gcd(b, a-b)
    */
    int g;
    g = y;
    while(x > 0){
        g = x;
        x = y % x;
        y = g;
    }
    return g;
}


int replace(char *s, char och, char nch){
    int nrep = 0;
    while(s = strchr(s, och)){
        *(s++) = nch;
        nrep++;
    }
    return nrep;
}

double distance(Point *a, Point *b){
    double dx, dy;
    dx = a->x - b->x;
    dy = a->y - b->y;
    return sqrt(dx * dx + dy * dy);
}
// 测试代码调用的，编译成执行文件执行：cl /EHsc /MD /utf-8 /nologo C.c /link /MACHINE:X64 /NOLOGO /OUT:main.exe  
// int main(){
//     printf("公约数是:%d\n", gcd(100,254));
//     char *str = "Hello this is a world!";
//     printf("替换次数:%d\n", replace(str, 'i','X'));
//     printf("替换结果:%s\n", str);
//     Point a = {0, 1};
//     Point b = {1, 0};
//     printf("距离:%f\n", distance(&a, &b));
//     return 0;
// }

定义模块

PyModuleDef结构体

typedef struct PyModuleDef{
  PyModuleDef_Base m_base;
  const char* m_name;
  const char* m_doc;
  Py_ssize_t m_size;
  PyMethodDef *m_methods;
  struct PyModuleDef_Slot* m_slots;
  traverseproc m_traverse;
  inquiry m_clear;
  freefunc m_free;
} PyModuleDef;

PyMethodDef结构体

struct PyMethodDef {
    const char  *ml_name;   /* The name of the built-in function/method */
    PyCFunction ml_meth;    /* The C function that implements it */
    int         ml_flags;   /* Combination of METH_xxx flags, which mostly
                               describe the args expected by the C func */
    const char  *ml_doc;    /* The __doc__ attribute, or NULL */
};
typedef struct PyMethodDef PyMethodDef;

模块定义的例子

static PyMethodDef _mymethos[] = {
    {"gcd",      py_gcd,                   METH_VARARGS,                 py_gcd_doc},
    {"replace",  (PyCFunction)py_replace,  METH_KEYWORDS | METH_VARARGS, py_replace_doc},    // 因为带三个参数，属于PyCFunctionWithKeywords
    {"distance", py_distance,              METH_VARARGS,                 py_distance_doc},
    {NULL,       NULL,                     0,                            NULL}
};


static struct PyModuleDef _mymodule = {
    PyModuleDef_HEAD_INIT,
    "mymodule",
    NULL,
    -1,
    _mymethos
};

创建模块

导出函数


PyMODINIT_FUNC PyInit_mymodule(void){
    return PyMODINIT_FUNC对象;
}

创建模块并返回


PyMODINIT_FUNC PyInit_mymodule(void){
    PyObject *mod;
    mod = PyModule_Create(&_mymodule);
    return mod;
}

Python的C扩展例子

C功能实现

C头文件:C.h 

#ifndef C_YQ_H
#define C_YQ_H
#include <stdio.h>
#include <string.h>
#include <math.h>

typedef struct Point{
    double x;
    double y;
} Point;


extern int gcd(int x, int y);
extern int replace(char *s, char och, char nch);
extern double distance(Point *a, Point *b);

#define MAGIC  0x31337

#endif

C实现文件:C.c 

#include "C.h"
int gcd(int x, int y){
    /* 
        这个算法是来自最大公约数的一个性质
        gcd(a,b)=gcd(b, a mod b)
        gcd(a,b)=gcd(b, a-b)
    */
    int g;
    g = y;
    while(x > 0){
        g = x;
        x = y % x;
        y = g;
    }
    return g;
}


int replace(char *s, char och, char nch){
    int nrep = 0;
    while(s = strchr(s, och)){
        *(s++) = nch;
        nrep++;
    }
    return nrep;
}

double distance(Point *a, Point *b){
    double dx, dy;
    dx = a->x - b->x;
    dy = a->y - b->y;
    return sqrt(dx * dx + dy * dy);
}
// 测试代码调用的，编译成执行文件执行：cl /EHsc /MD /utf-8 /nologo C.c /link /MACHINE:X64 /NOLOGO /OUT:main.exe  
// int main(){
//     printf("公约数是:%d\n", gcd(100,254));
//     char *str = "Hello this is a world!";
//     printf("替换次数:%d\n", replace(str, 'i','X'));
//     printf("替换结果:%s\n", str);
//     Point a = {0, 1};
//     Point b = {1, 0};
//     printf("距离:%f\n", distance(&a, &b));
//     return 0;
// }

C扩展实现

#include <Python.h>
#include "C.h"


static char py_gcd_doc[] = "最大公约数";
static PyObject* py_gcd(PyObject*self, PyObject *args){
    int r, x, y;
    // 解析参数
    if(!PyArg_ParseTuple(args, "ii:gcd", &x, &y)){
        return NULL;
    }
    r = gcd(x, y);
    return Py_BuildValue("i", r); 
}

static char py_replace_doc[] = "字符替换";
static PyObject* py_replace(PyObject* self, PyObject *args, PyObject *kwargs){
    char *s, *sdup;
    char och, nch;
    int nrep;

    PyObject *result;
    static char *argsname[] = {"s", "och", "nch", NULL};
    // 解析参数
    if(!PyArg_ParseTupleAndKeywords(args, kwargs, "scc:replace", argsname, &s, &och, &nch)){
        return NULL;
    }
    sdup = (char *)malloc(strlen(s) + 1);
    strcpy(sdup, s);
    nrep = replace(sdup, och, nch);
    result = Py_BuildValue("(is)", nrep, sdup);
    free(sdup);
    return result;
}

static char py_distance_doc[] = "计算距离";
static PyObject* py_distance(PyObject *self, PyObject *args){
    PyErr_SetString(PyExc_NotImplementedError, "distance() not implements");
    return NULL;
}

static PyMethodDef _mymethos[] = {
    {"gcd",      py_gcd,                   METH_VARARGS,                 py_gcd_doc},
    {"replace",  (PyCFunction)py_replace,  METH_KEYWORDS | METH_VARARGS, py_replace_doc},    // 因为带三个参数，属于PyCFunctionWithKeywords
    {"distance", py_distance,              METH_VARARGS,                 py_distance_doc},
    {NULL,       NULL,                     0,                            NULL}
};


static struct PyModuleDef _mymodule = {
    PyModuleDef_HEAD_INIT,
    "mymodule",
    NULL,
    -1,
    _mymethos
};

PyMODINIT_FUNC PyInit_mymodule(void){
    PyObject *mod;
    mod = PyModule_Create(&_mymodule);
    PyModule_AddIntMacro(mod, MAGIC);
    return mod;
}

编译脚本

  1. 脚本：Makefile
pyd: C.c C.h C_ython.c
# 切换utf-8编码
    @chcp 65001
# 编译C.c
    @cl /c /EHsc /MD /utf-8 /nologo /Fo:C.obj  C.c
# 编译C_ython.c
    @cl /c /EHsc /MD /utf-8 /nologo "-IC:\Program Files\Python36\include" /Fo:C_ython.obj  C_ython.c
# 链接动态库
    @link /MACHINE:X64 /NOLOGO /DLL  /OUT:mymodule.pyd  /EXPORT:PyInit_mymodule C.obj C_ython.obj
clean:
    @del *.obj *.dll *.pdb *.ilk *.exe *.lib  *.exp  *.pyd 2>/Nula
  1. 执行脚本：
    • vcvars64.bat
    • nmake pyd
  1. 编译效果
编译与结果截图

Python调用代码

# 模块导入
import mymodule

# 模块调用
print(mymodule.gcd(6, 8))

Python执行效果

Python调用C扩展模块的结果

观察下pyd的导出信息

Python的C扩展模块的导出信息

C扩展的核心

Python的扩展编译

扩展编译

setup.py文件模式

from distutils.core import setup, Extension
setup(
    name="my",
    version="1.1",
    ext_modules=[
        Extension("mymodule", sources=["C_ython.c", "C.c"],language="C"), 
    ]
)

setup函数

from distutils.core import setup, Extension
help(setup)

Help on function setup in module distutils.core:

setup(**attrs)
    The gateway to the Distutils: do everything your setup script needs
    to do, in a highly flexible and user-driven way.  Briefly: create a
    Distribution instance; find and parse config files; parse the command
    line; run each Distutils command found there, customized by the options
    supplied to 'setup()' (as keyword arguments), in config files, and on
    the command line.
    
    The Distribution instance might be an instance of a class supplied via
    the 'distclass' keyword argument to 'setup'; if no such class is
    supplied, then the Distribution class (in dist.py) is instantiated.
    All other arguments to 'setup' (except for 'cmdclass') are used to set
    attributes of the Distribution instance.
    
    The 'cmdclass' argument, if supplied, is a dictionary mapping command
    names to command classes.  Each command encountered on the command line
    will be turned into a command class, which is in turn instantiated; any
    class found in 'cmdclass' is used in place of the default, which is
    (for command 'foo_bar') class 'foo_bar' in module
    'distutils.command.foo_bar'.  The command class must provide a
    'user_options' attribute which is a list of option specifiers for
    'distutils.fancy_getopt'.  Any command-line options between the current
    and the next command are used to set attributes of the current command
    object.
    
    When the entire command-line has been successfully parsed, calls the
    'run()' method on each command object in turn.  This method will be
    driven entirely by the Distribution object (which each command object
    has a reference to, thanks to its constructor), and the
    command-specific options that became attributes of each command
    object.

setup.py的常见任务

C:\01works\13python\codes\cython_py>python setup.py --help-commands
Standard commands:
  build            build everything needed to install
  build_py         "build" pure Python modules (copy to build directory)
  build_ext        build C/C++ extensions (compile/link to build directory)
  build_clib       build C/C++ libraries used by Python extensions
  build_scripts    "build" scripts (copy and fixup #! line)
  clean            clean up temporary files from 'build' command
  install          install everything from build directory
  install_lib      install all Python modules (extensions and pure Python)
  install_headers  install C/C++ header files
  install_scripts  install scripts (Python or otherwise)
  install_data     install data files
  sdist            create a source distribution (tarball, zip file, etc.)
  register         register the distribution with the Python package index
  bdist            create a built (binary) distribution
  bdist_dumb       create a "dumb" built distribution
  bdist_rpm        create an RPM distribution
  bdist_wininst    create an executable installer for MS Windows
  check            perform some checks on the package
  upload           upload binary package to PyPI

usage: setup.py [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...]
   or: setup.py --help [cmd1 cmd2 ...]
   or: setup.py --help-commands
   or: setup.py cmd --help

build_ext的选项

C:\01works\13python\codes\cython_py>python setup.py build_ext --help
Common commands: (see '--help-commands' for more)

  setup.py build      will build the package underneath 'build/'
  setup.py install    will install the package

Global options:
  --verbose (-v)  run verbosely (default)
  --quiet (-q)    run quietly (turns verbosity off)
  --dry-run (-n)  don't actually do anything
  --help (-h)     show detailed help message
  --no-user-cfg   ignore pydistutils.cfg in your home directory

Options for 'build_ext' command:
  --build-lib (-b)     directory for compiled extension modules
  --build-temp (-t)    directory for temporary files (build by-products)
  --plat-name (-p)     platform name to cross-compile for, if supported
                       (default: win-amd64)
  --inplace (-i)       ignore build-lib and put compiled extensions into the
                       source directory alongside your pure Python modules
  --include-dirs (-I)  list of directories to search for header files
                       (separated by ';')
  --define (-D)        C preprocessor macros to define
  --undef (-U)         C preprocessor macros to undefine
  --libraries (-l)     external C libraries to link with
  --library-dirs (-L)  directories to search for external C libraries
                       (separated by ';')
  --rpath (-R)         directories to search for shared C libraries at runtime
  --link-objects (-O)  extra explicit link objects to include in the link
  --debug (-g)         compile/link with debugging information
  --force (-f)         forcibly build everything (ignore file timestamps)
  --compiler (-c)      specify the compiler type
  --parallel (-j)      number of parallel build jobs
  --swig-cpp           make SWIG create C++ files (default is C)
  --swig-opts          list of SWIG command line options
  --swig               path to the SWIG executable
  --user               add user include, library and rpath
  --help-compiler      list available compilers

usage: setup.py [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...]
   or: setup.py --help [cmd1 cmd2 ...]
   or: setup.py --help-commands
   or: setup.py cmd --help

Extension类

from distutils.core import setup, Extension
help(Extension)

Help on class Extension in module distutils.extension:

class Extension(builtins.object)
 |  Just a collection of attributes that describes an extension
 |  module and everything needed to build it (hopefully in a portable
 |  way, but there are hooks that let you be as unportable as you need).
 |  
 |  Instance attributes:
 |    name : string
 |      the full name of the extension, including any packages -- ie.
 |      *not* a filename or pathname, but Python dotted name
 |    sources : [string]
 |      list of source filenames, relative to the distribution root
 |      (where the setup script lives), in Unix form (slash-separated)
 |      for portability.  Source files may be C, C++, SWIG (.i),
 |      platform-specific resource files, or whatever else is recognized
 |      by the "build_ext" command as source for a Python extension.
 |    include_dirs : [string]
 |      list of directories to search for C/C++ header files (in Unix
 |      form for portability)
 |    define_macros : [(name : string, value : string|None)]
 |      list of macros to define; each macro is defined using a 2-tuple,
 |      where 'value' is either the string to define it to or None to
 |      define it without a particular value (equivalent of "#define
 |      FOO" in source or -DFOO on Unix C compiler command line)
 |    undef_macros : [string]
 |      list of macros to undefine explicitly
 |    library_dirs : [string]
 |      list of directories to search for C/C++ libraries at link time
 |    libraries : [string]
 |      list of library names (not filenames or paths) to link against
 |    runtime_library_dirs : [string]
 |      list of directories to search for C/C++ libraries at run time
 |      (for shared extensions, this is when the extension is loaded)
 |    extra_objects : [string]
 |      list of extra files to link with (eg. object files not implied
 |      by 'sources', static library that must be explicitly specified,
 |      binary resource files, etc.)
 |    extra_compile_args : [string]
 |      any extra platform- and compiler-specific information to use
 |      when compiling the source files in 'sources'.  For platforms and
 |      compilers where "command line" makes sense, this is typically a
 |      list of command-line arguments, but for other platforms it could
 |      be anything.
 |    extra_link_args : [string]
 |      any extra platform- and compiler-specific information to use
 |      when linking object files together to create the extension (or
 |      to create a new static Python interpreter).  Similar
 |      interpretation as for 'extra_compile_args'.
 |    export_symbols : [string]
 |      list of symbols to be exported from a shared extension.  Not
 |      used on all platforms, and not generally necessary for Python
 |      extensions, which typically export exactly one symbol: "init" +
 |      extension_name.
 |    swig_opts : [string]
 |      any extra options to pass to SWIG if a source file has the .i
 |      extension.
 |    depends : [string]
 |      list of files that the extension depends on
 |    language : string
 |      extension language (i.e. "c", "c++", "objc"). Will be detected
 |      from the source extensions if not provided.
 |    optional : boolean
 |      specifies that a build failure in the extension should not abort the
 |      build process, but simply not install the failing extension.
 |  
 |  Methods defined here:
 |  
 |  __init__(self, name, sources, include_dirs=None, define_macros=None, undef_macros=None, library_dirs=None, libraries=None, runtime_library_dirs=None, extra_objects=None, extra_compile_args=None, extra_link_args=None, export_symbols=None, swig_opts=None, depends=None, language=None, optional=None, **kw)
 |      Initialize self.  See help(type(self)) for accurate signature.
 |  
 |  __repr__(self)
 |      Return repr(self).
 |  
 |  ----------------------------------------------------------------------
 |  Data descriptors defined here:
 |  
 |  __dict__
 |      dictionary for instance variables (if defined)
 |  
 |  __weakref__
 |      list of weak references to the object (if defined)

扩展编译结果

使用python的扩展编译实现编译

包装扩展模块的Python模块

  1. 包装模块
    • 文件名：my.py 
from mymodule import *
  1. 调用模块
    • test.py
import my
print(my.gcd(6, 8))

安装

安装的目录结构

setup.py需要安装内容的路径

安装脚本文件setup.py

from distutils.core import setup, Extension
setup(
    name="mydemos",  # 用于pip安装中指定的名字
    version="1.1",
    packages=["mydemo", "mydemo.mypkg"],   # 包路径下一定要有__init__.py
    scripts=["test.py"],
    ext_modules=[
        Extension("mymodule", sources=["C_ython.c", "C.c"],language="C"),    # 需要安装的python扩展模块
    ]
)

直接安装

安装过程截图 安装结果 pip list查看 使用安装好的模块

安装打包

制作安装包的过程

附录

上一篇 下一篇

猜你喜欢

热点阅读