My Project
mlpredict.c
Go to the documentation of this file.
1 /**
2  * @file mlpredict.c
3  * @brief Functions for using python to do machine learning in Singular
4  *
5  * @author Murray Heymann
6  * @date August 2019
7  */
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <string.h>
11 
12 #include "kernel/mod2.h"
13 #if defined(HAVE_READLINE) && defined(HAVE_READLINE_READLINE_H)
14 #ifdef HAVE_PYTHON
15 
16 #include <Python.h>
17 
18 #include "mlpredict.h"
19 
20 /* Locally defined macros */
21 #define LOOKUPTABLE "common.lookuptable"
22 #define KEYWORD_VECTOR "common.keyword_vector"
23 #define PRD_RUNNER "predictor_runner"
24 #define IS_LOOKUP_INITIALISED "is_lookup_initialised"
25 #define INIT_TABLE_ON_SYSTEM "init_table_on_system"
26 #define GET_PREDICTION "get_prediction"
27 #define READ_DICTIONARY "read_dictionary"
28 #define CREATE_TABLE "create_table"
29 #define PYTPATH(B) sprintf(B, "%s/ml_python", DATA_PATH)
30 #define SING_BIN(B) sprintf(B, "%s/Singular", BIN_PATH)
31 #define SING_EXT_SCRIPT(B) sprintf(B, "%s/ml_singular/extract.lib", \
32  DATA_PATH)
33 
34 /**** Local Function Declarations ****************************************/
35 
36 PyObject *_call_python_function(char *module, char *func);
37 PyObject *_call_python_function_args(char *module, char *func, PyObject *pArgs);
38 ml_internal *_get_internals();
39 int _set_dictionary();
40 int _set_vectors_file_list();
41 
42 /**** Static Variables ***************************************************/
43 
44 //static PyObject *pDictionary = NULL;
45 //static PyObject *pVectors = NULL;
46 //static PyObject *pFile_list = NULL;
47 static ml_internal internal_obs = {NULL,NULL,NULL};
48 
49 /**** Public Functions ***************************************************/
50 
51 /**
52  * Check whether the helpfiles have been downloaded and the relevant
53  * vectors have been calculated and saved. Furthermore, the local static
54  * variables must have been set with pointers to the relevant data
55  * structures in the python instance.
56  *
57  * @return An integer: 1 if it has been intialised, 0 otherwise
58  */
59 int ml_is_initialised()
60 {
61  int t_value = 0;
62  PyObject *pValue = NULL;
63 
64  if (!Py_IsInitialized()) return 0;
65 
66  pValue = _call_python_function(LOOKUPTABLE, IS_LOOKUP_INITIALISED);
67 
68  if (pValue == NULL) return 0;
69  if (!PyBool_Check(pValue)) {
70  /* Not a boolean type */
71  Py_DECREF(pValue);
72  return 0;
73  }
74 
75  t_value = PyObject_IsTrue(pValue);
76  Py_DECREF(pValue);
77  if (t_value == -1) {
78  /* errors */
79  PyErr_Print();
80  Py_DECREF(pValue);
81  return 0;
82  }
83 
84  if (!t_value) return 0;
85  if (!internal_obs.pDictionary) return 0;
86  if (!internal_obs.pVectors) return 0;
87  if (!internal_obs.pFile_list) return 0;
88 
89  return 1;
90 }
91 
92 /**
93  * Initialise the machine learning system by starting the python
94  * interpreter, downloading the helpfiles if
95  * not present, and calculating the bag of words vectors for the helpfiles,
96  * saving this info on the local system.
97  *
98  * @return An integer: 1 if successful, 0 if some error were to occur.
99  */
100 int ml_initialise()
101 {
102  char buffer[100];
103  char *spath = NULL;
104  PyObject *pString = NULL;
105  PyObject *pValue = NULL;
106  PyObject *pPath = NULL;
107  PyObject *pMyPath = NULL;
108 
109  PyObject *pName = NULL;
110  PyObject *pModule = NULL;
111  PyObject *pArgs = NULL;
112  //PyObject *pTemp = NULL;
113 
114  if (!Py_IsInitialized()) {
115  Py_Initialize();
116  }
117 
118  pName = PyString_FromString("sys");
119  //pName = PyUnicode_FromString("sys");
120  pModule = PyImport_Import(pName);
121  Py_DECREF(pName);
122  if (pModule == NULL){
123  PyErr_Print();
124  fprintf(stderr, "Failed to load \"%s\"\n", "sys");
125  return 0;
126  }
127 
128  /* Get the path list */
129  pPath = PyObject_GetAttrString(pModule, "path");
130 
131  if (!pPath) {
132  fprintf(stderr, "Failed to get python path list\n");
133  Py_DECREF(pModule);
134  return 0;
135  }
136  Py_DECREF(pModule);
137 
138  /* get a string representation of the path list for comparison */
139  pString = PyObject_Str(pPath);
140  spath = PyString_AsString(pString);
141  //pTemp = PyUnicode_AsASCIIString(pString);
142  //spath = PyBytes_AsString(pTemp);
143  /* get the path to be set */
144  PYTPATH(buffer);
145  if (!strstr(spath, buffer)) {
146  pMyPath = PyString_FromString(buffer);
147  //pMyPath = PyUnicode_FromString(buffer);
148  /* pPath set to tuple, so no decref needed later */
149  PyList_Append(pPath, pMyPath);
150  Py_DECREF(pMyPath);
151  }
152  Py_DECREF(pString);
153  //Py_XDECREF(pTemp);
154  Py_DECREF(pPath);
155 
156 
157  /* Setup arguments */
158  pArgs = PyTuple_New(2);
159  SING_EXT_SCRIPT(buffer);
160  pString = PyString_FromString(buffer);
161  PyTuple_SetItem(pArgs, 0, pString);
162 
163  SING_BIN(buffer);
164  pString = PyString_FromString(buffer);
165  PyTuple_SetItem(pArgs, 1, pString);
166 
167  pValue = _call_python_function_args(LOOKUPTABLE,
168  INIT_TABLE_ON_SYSTEM,
169  pArgs);
170  Py_DECREF(pArgs);
171 
172  if (pValue == NULL) return 0;
173  Py_XDECREF(pValue);
174  if (!_set_dictionary()) return 0;
175  if (!_set_vectors_file_list()) return 0;
176 
177  return 1;
178 }
179 
180 /**
181  * Tell python to decrement the global variables, checking whether it is
182  * necessary in the first place.
183  *
184  * @return An integer: 1 if successful, 0 if not.
185  */
186 int ml_finalise()
187 {
188  if (!Py_IsInitialized())
189  return 0;
190 
191  Py_XDECREF(internal_obs.pDictionary);
192  Py_XDECREF(internal_obs.pVectors);
193  Py_XDECREF(internal_obs.pFile_list);
194  internal_obs.pDictionary = NULL;
195  internal_obs.pVectors = NULL;
196  internal_obs.pFile_list = NULL;
197 
198  /* Note Py_Finalize should only be called when the program quits. not
199  * here.
200  */
201  return 1;
202 }
203 
204 /**
205  * Take a filename as string, pass it to the machine learning system, and
206  * return a helpfile name as string.
207  *
208  * @param[in] filename A String indicating the for which the prediction
209  * must be made
210  * @param[out] prediction_buffers The buffer into which the prediction
211  * filenames are copied. Must contain exactly 5 char *.
212  * @param[out] pred_len A pointer to an integer, at which the string length
213  * of the prediction filename is set.
214  *
215  * @return 1 if successful, 0 if some error occurs.
216  */
217 int ml_make_prediction(char *filename,
218  char *prediction_buffers[],
219  int *pred_len,
220  char *(*custom_strdup)(const char *))
221 {
222  PyObject *pFName = NULL;
223  PyObject *pArgs = NULL;
224  PyObject *pValue = NULL;
225  PyObject *pString = NULL;
226  // PyObject *pTemp = NULL;
227  int i = 0;
228 
229  pFName = PyString_FromString(filename);
230  //pFName = PyUnicode_FromString(filename);
231  if (!pFName) {
232  fprintf(stderr, "This is weird\n");
233 
234  return 0;
235  }
236  pArgs = PyTuple_New(4);
237  if (!pArgs) {
238  fprintf(stderr, "This is also weird\n");
239  Py_DECREF(pFName);
240  return 0;
241  }
242  /* pFName is handed over to the tuple, so not DECREF later */
243  PyTuple_SetItem(pArgs, 0, pFName);
244  /* Since each of the following is handed over, we need to increase the
245  * reference, otherwise our static variable pointers might be freed by
246  * the python interpreter. */
247  PyTuple_SetItem(pArgs, 1, internal_obs.pDictionary);
248  Py_INCREF(internal_obs.pDictionary);
249  PyTuple_SetItem(pArgs, 2, internal_obs.pVectors);
250  Py_INCREF(internal_obs.pVectors);
251  PyTuple_SetItem(pArgs, 3, internal_obs.pFile_list);
252  Py_INCREF(internal_obs.pFile_list);
253 
254  pValue = _call_python_function_args(PRD_RUNNER,
255  GET_PREDICTION,
256  pArgs);
257  Py_DECREF(pArgs);
258 
259  if (!pValue) {
260  return 0;
261  }
262  if (!PyList_Check(pValue)) {
263  printf("Expected a list for prediction.\n");
264  Py_DECREF(pValue);
265  return 0;
266  }
267  if (PyList_Size(pValue) != 5) {
268  printf("List length is supposed to be five, but is %d.\n",
269  (int)PyList_Size(pValue));
270  Py_DECREF(pValue);
271  return 0;
272  }
273  // pString = PyObject_Str(pValue);
274  for (i = 0; i < 5; i++) {
275  pString = PyObject_Str(PyList_GetItem(pValue, i));
276  prediction_buffers[i] = custom_strdup(PyString_AsString(pString));
277  //pTemp = PyUnicode_AsASCIIString(pString);
278  //prediction_buffers[i] = custom_strdup(PyBytes_AsString(pTemp));
279  pred_len[i] = strlen(prediction_buffers[i]);
280  Py_DECREF(pString);
281  pString = NULL;
282  }
283 
284  Py_DECREF(pValue);
285 
286  return 1;
287 }
288 
289 /**** Local Functions ****************************************************/
290 
291 /**
292  * Local helper function to call a function that takes no arguments.
293  * @param[in] module A string of the module name where the function is
294  * found
295  * @param[in] func A string giving the name of the function to call.
296  *
297  * @return the returned PyObject.
298  */
299 PyObject *_call_python_function(char *module, char *func)
300 {
301  PyObject *pArgs = PyTuple_New(0);
302  PyObject *retvalue = _call_python_function_args(module, func, pArgs);
303 
304  Py_DECREF(pArgs);
305  return retvalue;
306 }
307 
308 
309 /**
310  * Local helper function to call a function that takes arguments.
311  * @param[in] module A string of the module name where the function is
312  * found
313  * @param[in] func A string giving the name of the function to call.
314  * @param[in] pArgs The arguments to be parsed to the funcion being called.
315  *
316  * @return the returned PyObject.
317  */
318 PyObject *_call_python_function_args(char *module, char *func, PyObject *pArgs)
319 {
320  PyObject *pName = NULL, *pModule = NULL, *pFunc = NULL;
321  PyObject *pValue = NULL;
322 
323  if (!Py_IsInitialized()) {
324  Py_Initialize();
325  printf("I don't like this\n");
326  }
327 
328  /* import the module */
329  pName = PyString_FromString(module);
330  //pName = PyUnicode_FromString(module);
331  pModule = PyImport_Import(pName);
332  Py_DECREF(pName);
333 
334  if (pModule == NULL){
335  PyErr_Print();
336  fprintf(stderr, "Failed to load \"%s\"\n", module);
337  return NULL;
338  }
339  /* Get the init function we want to call */
340  pFunc = PyObject_GetAttrString(pModule, func);
341  if (!pFunc || !PyCallable_Check(pFunc)) {
342  /* Somehow not executable. Clean up! */
343  if(PyErr_Occurred()) {
344  PyErr_Print();
345  }
346  fprintf(stderr,
347  "Cannot find function \"%s\"\n",
348  func);
349 
350  Py_XDECREF(pFunc);
351  Py_DECREF(pModule);
352 
353  return NULL;
354  }
355 
356  /* Callable function. Good. Call with the args supplied,
357  * assuming the arguments are correct for the function */
358  pValue = PyObject_CallObject(pFunc, pArgs);
359 
360  Py_XDECREF(pFunc);
361  Py_DECREF(pModule);
362 
363  if (pValue == NULL) {
364  printf("No return for function\n");
365  PyErr_Print();
366  return NULL;
367  }
368  return pValue;
369 }
370 
371 /**
372  * This is intended to be used ONLY by the testing library.
373  */
374 ml_internal *_get_internals()
375 {
376  return &internal_obs;
377 }
378 
379 
380 /**
381  * Get the PyObject of the list of keywords called dictionary and set it to
382  * the local static variable. If already set, simply return without any
383  * action.
384  *
385  * @return 1 if successful, 0 if something goes wrong.
386  */
387 int _set_dictionary()
388 {
389  PyObject *pValue = NULL;
390 
391  if (internal_obs.pDictionary) {
392  /* already set */
393  return 1;
394  }
395  pValue = _call_python_function(KEYWORD_VECTOR, READ_DICTIONARY);
396  if (!pValue) {
397  return 0;
398  }
399  internal_obs.pDictionary = pValue;
400 
401  return 1;
402 }
403 
404 /**
405  * Get the PyObject of the list of vectors called corresponding to each
406  * helper file and the list of helper files that could be in a prediction
407  * and set it to the local static variable. If already set, simply return
408  * without any action.
409  *
410  * @return 1 if successful, 0 if something goes wrong.
411  */
412 
413 int _set_vectors_file_list()
414 {
415  PyObject *pValue = NULL;
416  PyObject *pVal1 = NULL, *pVal2 = NULL;
417 
418  if (internal_obs.pVectors && internal_obs.pFile_list) {
419  return 1;
420  }
421 
422  /* Ensure *both* are free */
423  Py_XDECREF(internal_obs.pVectors);
424  Py_XDECREF(internal_obs.pFile_list);
425  internal_obs.pVectors = NULL;
426  internal_obs.pFile_list = NULL;
427 
428  pValue = _call_python_function(LOOKUPTABLE, CREATE_TABLE);
429  if (!pValue) {
430  return 0;
431  }
432  pVal1 = PyTuple_GetItem(pValue, 0);
433  pVal2 = PyTuple_GetItem(pValue, 1);
434  /* Decreasing the reference to the tuple causes the content to be freed.
435  * To prevent that (leading to a segfault), we have to manually increase
436  * the reference to the contents */
437  Py_INCREF(pVal1);
438  Py_INCREF(pVal2);
439 
440  Py_DECREF(pValue);
441  pValue = NULL;
442 
443  if (!pVal1 || !pVal2) {
444  Py_XDECREF(pVal1);
445  Py_XDECREF(pVal2);
446  return 0;
447  }
448  internal_obs.pVectors = pVal1;
449  internal_obs.pFile_list = pVal2;
450 
451  return 1;
452 }
453 #endif
454 #endif
int i
Definition: cfEzgcd.cc:132
VAR clock_t t_value
Definition: cohomo.cc:3182
Function definitions for using python to do machine learning in Singular.
#define NULL
Definition: omList.c:12
char * pString(poly p)
Definition: polys.h:306
static CanonicalForm * retvalue
Definition: readcf.cc:126