From 440561fc8e0b9582b5dd3debbf7f6844ae8d81a0 Mon Sep 17 00:00:00 2001 From: Alexis de Lattre Date: Fri, 14 Oct 2016 11:48:54 +0200 Subject: [PATCH 1/6] report_py3o: full installation procedure in README --- report_py3o/README.rst | 128 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 121 insertions(+), 7 deletions(-) diff --git a/report_py3o/README.rst b/report_py3o/README.rst index cf4d70620a..bda1661714 100644 --- a/report_py3o/README.rst +++ b/report_py3o/README.rst @@ -34,20 +34,134 @@ You must install 2 additionnal python libs: pip install py3o.template pip install py3o.formats -If you want to convert the ODT or ODS report in another format, you need several additionnal components and Python libs: +To allow the conversion of ODT or ODS reports to other formats (PDF, DOC, DOCX, etc.), you must install several additionnal components and Python libs: -* `Py3o Fusion server `_ -* `Py3o render server `_ -* Libreoffice started in the background in headless mode. +* `Py3o Fusion server `_, +* `Py3o render server `_, +* a Java Runtime Environment (JRE), which can be OpenJDK, +* Libreoffice started in the background in headless mode, +* the Java driver for Libreoffice (Juno). -TODO : continue +It is also possible to use the Python driver for Libreoffice (PyUNO), but it is recommended to use the Java driver because it is more stable. + +The installation procedure below uses the Java driver. It has been successfully tested on Ubuntu 16.04 LTS ; if you use another OS, you may have to change a few details. + +Installation of py3o.fusion: + +.. code:: + + pip install py3o.fusion + pip install service-identity + +Installation of py3o.renderserver: + +.. code:: + + pip install py3o.renderserver + +Installation of Libreoffice and JRE on Debian/Ubuntu: + +.. code:: + + sudo apt-get install default-jre ure libreoffice-java-common libreoffice-writer + +At the end, with the dependencies, you should have 6 py3o python libs: + +.. code:: + + % pip freeze | grep py3o + py3o.formats==0.3 + py3o.fusion==0.8.6.dev1 + py3o.renderclient==0.2 + py3o.renderers.juno==0.7 + py3o.renderserver==0.5.1.dev1 + py3o.template==0.9.10.dev1 + py3o.types==0.1.1 + +Start the Py3o Fusion server: + +.. code:: + + start-py3o-fusion --debug -s localhost + +Start the Py3o render server: + +.. code:: + + start-py3o-renderserver --java=/usr/lib/jvm/default-java/jre/lib/amd64/server/libjvm.so --ure=/usr/lib --office=/usr/lib/libreoffice --driver=juno --sofficeport=8997 + +On the output of the Py3o render server, the first line looks like: + +.. code:: + + DEBUG:root:Starting JVM: /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server/libjvm.so with options: -Djava.class.path=/usr/local/lib/python2.7/dist-packages/py3o/renderers/juno/py3oconverter.jar:/usr/lib/ure/share/java/juh.jar:/usr/lib/ure/share/java/jurt.jar:/usr/lib/ure/share/java/ridl.jar:/usr/lib/ure/share/java/unoloader.jar:/usr/lib/ure/share/java/java_uno.jar:/usr/lib/libreoffice/program/classes/unoil.jar -Xmx150M + +After **-Djava.class.path**, there is a list of Java libs with *.jar* extension ; check that each JAR file is really present on your filesystem. On Ubuntu 16.04 LTS, the package *ure* installs several libs in another directory: + +* /usr/lib/ure/share/java/juh.jar is located in /usr/share/java/juh.jar +* /usr/lib/ure/share/java/jurt.jar is located in /usr/share/java/jurt.jar +* /usr/lib/ure/share/java/ridl.jar is located in /usr/share/java/ridl.jar +* /usr/lib/ure/share/java/unoloader.jar is located in /usr/share/java/unoloader.jar +* /usr/lib/ure/share/java/java_uno.jar is located in /usr/share/java/java_uno.jar + +To work around this problem, you can create a symlink: + +.. code:: + + sudo ln -s /usr /ure + +and then use **--ure=/** instead of **--ure=/usr/lib** in the command line of *start-py3o-renderserver*. + +To check that the Py3o Fusion server is running fine, visit the URL http://:8765/form. On this web page, under the section *Target format*, make sure that you have a line *This server currently supports these formats: ods, odt, docx, doc, html, docbook, pdf, xls.*. Configuration ============= -If you want to convert the report in another format, go to the menu *Configuration > Technical > Reports > Py3o > Py3o Servers* and create a new Py3o server with its URL (for example: http://localhost:8765/form). +For example, to replace the native invoice report by a custom py3o report, add the following XML file in your custom module: + +.. code:: + + + + + + Invoice + account.invoice + account.report_invoice + py3o + odt + my_custom_module_base + report/account_invoice.odt + + + + +where *my_custom_module_base* is the name of the custom Odoo module. In this example, the invoice ODT file is located in *my_custom_module_base/report/account_invoice.odt*. + +If you want an invoice in PDF format instead of ODT format, the XML file should look like: + +.. code:: + + + + + + http://localhost:8765/form + + + + Invoice + account.invoice + account.report_invoice + py3o + pdf + + my_custom_module_base + report/account_invoice.odt + + + -TODO: continue Usage ===== From 833fd479c7360e86864f5690c2090618ac3db03a Mon Sep 17 00:00:00 2001 From: Alexis de Lattre Date: Wed, 19 Oct 2016 12:53:35 +0200 Subject: [PATCH 2/6] Add command line for libreoffice in README --- report_py3o/README.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/report_py3o/README.rst b/report_py3o/README.rst index bda1661714..34fdcaf3ba 100644 --- a/report_py3o/README.rst +++ b/report_py3o/README.rst @@ -8,7 +8,7 @@ Report Py3o The py3o reporting engine is a reporting engine for Odoo based on `Libreoffice `_: -* the report is created with Libreoffice (ODT or ODS), +* the report is created with Libreoffice (ODT or ODS files) or any other software that generate files in `OpenDocument `_ format, * the report is stored on the server in OpenDocument format (.odt or .ods file) * the report is sent to the user in OpenDocument format or in any output format supported by Libreoffice (PDF, HTML, DOC, DOCX, Docbook, XLS, etc.) @@ -112,6 +112,12 @@ To work around this problem, you can create a symlink: and then use **--ure=/** instead of **--ure=/usr/lib** in the command line of *start-py3o-renderserver*. +Start Libreoffice in headless mode: + +.. code:: + + libreoffice --nologo --norestore --invisible --headless --nocrashreport --nofirststartwizard --nodefault --accept="socket,host=localhost,port=8997;urp;" + To check that the Py3o Fusion server is running fine, visit the URL http://:8765/form. On this web page, under the section *Target format*, make sure that you have a line *This server currently supports these formats: ods, odt, docx, doc, html, docbook, pdf, xls.*. Configuration From b74094587331c128f9a7cd448af3fadf5d1cf2ed Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Wed, 16 Nov 2016 14:57:14 +0100 Subject: [PATCH 3/6] [IMP] Escape 'False' when rendering value --- report_py3o/py3o_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/report_py3o/py3o_parser.py b/report_py3o/py3o_parser.py index 853035acbf..ec92bb82d5 100644 --- a/report_py3o/py3o_parser.py +++ b/report_py3o/py3o_parser.py @@ -148,7 +148,7 @@ def create_single_pdf(self, cr, uid, ids, data, report_xml, context=None): in_stream = StringIO(tmpl_data) out_stream = StringIO() - template = Template(in_stream, out_stream) + template = Template(in_stream, out_stream, escape_false=True) localcontext = parser_instance.localcontext if report_xml.py3o_is_local_fusion: template.render(localcontext) From 92f74f34877189b224cfd7711cef79478e6e0da5 Mon Sep 17 00:00:00 2001 From: Laurent Mignon Date: Sat, 17 Dec 2016 10:07:47 +0100 Subject: [PATCH 4/6] [IMP] Replace old style parser by TransientModel The goal is to improve the modularity by making the parser a true inheritable odoo model and share part of the code with the 'report' model --- report_py3o/models/__init__.py | 1 + report_py3o/models/ir_actions_report_xml.py | 59 ++--- .../{py3o_parser.py => models/py3o_report.py} | 202 ++++++++++++------ report_py3o/tests/test_report_py3o.py | 9 +- 4 files changed, 153 insertions(+), 118 deletions(-) rename report_py3o/{py3o_parser.py => models/py3o_report.py} (51%) diff --git a/report_py3o/models/__init__.py b/report_py3o/models/__init__.py index a8e7b0a69c..425cb3d9f0 100644 --- a/report_py3o/models/__init__.py +++ b/report_py3o/models/__init__.py @@ -1,3 +1,4 @@ from . import ir_actions_report_xml from . import py3o_template from . import py3o_server +from . import py3o_report diff --git a/report_py3o/models/ir_actions_report_xml.py b/report_py3o/models/ir_actions_report_xml.py index bce6593d8d..3f3323010d 100644 --- a/report_py3o/models/ir_actions_report_xml.py +++ b/report_py3o/models/ir_actions_report_xml.py @@ -1,13 +1,10 @@ # -*- coding: utf-8 -*- # Copyright 2013 XCG Consulting (http://odoo.consulting) # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). -import os import logging -from openerp import api, fields, models, SUPERUSER_ID, _ -from openerp.report.interface import report_int +from openerp import api, fields, models, _ from openerp.exceptions import ValidationError -from openerp import addons -from ..py3o_parser import Py3oParser + logger = logging.getLogger(__name__) @@ -85,44 +82,14 @@ def _get_py3o_filetypes(self): )) report_type = fields.Selection(selection_add=[('py3o', "Py3o")]) - @api.cr - def _lookup_report(self, cr, name): - """Look up a report definition. - """ - - # First lookup in the deprecated place, because if the report - # definition has not been updated, it is more likely the correct - # definition is there. Only reports with custom parser - # specified in Python are still there. - if 'report.' + name in report_int._reports: - new_report = report_int._reports['report.' + name] - if not isinstance(new_report, Py3oParser): - new_report = None - else: - report_data = self.search_read( - cr, SUPERUSER_ID, - [("report_name", "=", name), - ("report_type", "=", "py3o")], - ['parser', 'model', 'report_name', 'report_rml', 'header'], - limit=1) - if report_data: - report_data = report_data[0] - kwargs = {} - if report_data['parser']: - kwargs['parser'] = getattr(addons, report_data['parser']) - - new_report = Py3oParser( - 'report.' + report_data['report_name'], - report_data['model'], - os.path.join('addons', report_data['report_rml'] or '/'), - header=report_data['header'], - register=False, - **kwargs - ) - else: - new_report = None - - if new_report: - return new_report - else: - return super(IrActionsReportXml, self)._lookup_report(cr, name) + @api.model + def render_report(self, res_ids, name, data): + action_py3o_report = self.search( + [("report_name", "=", name), + ("report_type", "=", "py3o")]) + if action_py3o_report: + return self.env['py3o.report'].create({ + 'ir_actions_report_xml_id': action_py3o_report.id + }).create_report(res_ids, data) + return super(IrActionsReportXml, self).render_report( + res_ids, name, data) diff --git a/report_py3o/py3o_parser.py b/report_py3o/models/py3o_report.py similarity index 51% rename from report_py3o/py3o_parser.py rename to report_py3o/models/py3o_report.py index ec92bb82d5..9b57a703e0 100644 --- a/report_py3o/py3o_parser.py +++ b/report_py3o/models/py3o_report.py @@ -1,25 +1,29 @@ # -*- coding: utf-8 -*- # Copyright 2013 XCG Consulting (http://odoo.consulting) +# Copyright 2016 ACSONE SA/NV # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +import base64 +from base64 import b64decode from cStringIO import StringIO import json -import pkg_resources +import logging import os -import sys -from base64 import b64decode +import pkg_resources import requests +import sys from tempfile import NamedTemporaryFile -from openerp import _ -from openerp import exceptions -from openerp.report.report_sxw import report_sxw -from openerp import registry -import logging +from zipfile import ZipFile, ZIP_DEFLATED + +from openerp.exceptions import UserError +from openerp.report.report_sxw import rml_parse +from openerp import api, fields, models, _ logger = logging.getLogger(__name__) try: from py3o.template.helpers import Py3oConvertor from py3o.template import Template + from py3o import formats except ImportError: logger.debug('Cannot import py3o.template') try: @@ -64,11 +68,18 @@ def defautl_extend(report_xml, localcontext): localcontext['b64decode'] = b64decode -class Py3oParser(report_sxw): - """Custom class that use Py3o to render libroffice reports. - Code partially taken from CampToCamp's webkit_report.""" +class Py3oReport(models.TransientModel): + _name = "py3o.report" + _inherit = 'report' + _description = "Report Py30" - def get_template(self, report_obj): + ir_actions_report_xml_id = fields.Many2one( + comodel_name="ir.actions.report.xml", + required=True + ) + + @api.multi + def get_template(self): """private helper to fetch the template data either from the database or from the default template file provided by the implementer. @@ -76,30 +87,27 @@ def get_template(self, report_obj): to try and fetch the report template from database. If not found it will fallback to the template file referenced in the report definition. - @param report_obj: a recordset representing the report defintion - @type report_obj: openerp.model.recordset instance - @returns: string or buffer containing the template data @raises: TemplateNotFound which is a subclass of openerp.exceptions.DeferredException """ - + self.ensure_one() tmpl_data = None - - if report_obj.py3o_template_id and report_obj.py3o_template_id.id: + report_xml = self.ir_actions_report_xml_id + if report_xml.py3o_template_id and report_xml.py3o_template_id.id: # if a user gave a report template tmpl_data = b64decode( - report_obj.py3o_template_id.py3o_template_data + report_xml.py3o_template_id.py3o_template_data ) - elif report_obj.py3o_template_fallback: - tmpl_name = report_obj.py3o_template_fallback + elif report_xml.py3o_template_fallback: + tmpl_name = report_xml.py3o_template_fallback flbk_filename = None - if report_obj.module: + if report_xml.module: # if the default is defined flbk_filename = pkg_resources.resource_filename( - "openerp.addons.%s" % report_obj.module, + "openerp.addons.%s" % report_xml.module, tmpl_name, ) elif os.path.isabs(tmpl_name): @@ -119,37 +127,54 @@ def get_template(self, report_obj): return tmpl_data - def _extend_parser_context(self, parser_instance, report_xml): + @api.multi + def _extend_parser_context(self, context_instance, report_xml): # add default extenders for fct in _extender_functions.get(None, []): - fct(report_xml, parser_instance.localcontext) + fct(report_xml, context_instance.localcontext) # add extenders for registered on the template xml_id = report_xml.get_external_id().get(report_xml.id) if xml_id in _extender_functions: for fct in _extender_functions[xml_id]: - fct(report_xml, parser_instance.localcontext) + fct(report_xml, context_instance.localcontext) + + @api.multi + def _get_parser_context(self, model_instance, data): + report_xml = self.ir_actions_report_xml_id + context_instance = rml_parse(self.env.cr, self.env.uid, + report_xml.name, + context=self.env.context) + context_instance.set_context(model_instance, data, model_instance.ids, + report_xml.report_type) + self._extend_parser_context(context_instance, report_xml) + return context_instance.localcontext + + @api.multi + def _postprocess_report(self, content, res_id, save_in_attachment): + if save_in_attachment.get(res_id): + attachment = { + 'name': save_in_attachment.get(res_id), + 'datas': base64.encodestring(content), + 'datas_fname': save_in_attachment.get(res_id), + 'res_model': save_in_attachment.get('model'), + 'res_id': res_id, + } + return self.env['ir.attachment'].create(attachment) + return False - def create_single_pdf(self, cr, uid, ids, data, report_xml, context=None): - """ Overide this function to generate our py3o report + @api.multi + def _create_single_report(self, model_instance, data, save_in_attachment): + """ This function to generate our py3o report """ - if report_xml.report_type != 'py3o': - return super(Py3oParser, self).create_single_pdf( - cr, uid, ids, data, report_xml, context=context - ) + self.ensure_one() + report_xml = self.ir_actions_report_xml_id - parser_instance = self.parser(cr, uid, self.name2, context=context) - parser_instance.set_context( - self.getObjects(cr, uid, ids, context), - data, ids, report_xml.report_type - ) - self._extend_parser_context(parser_instance, report_xml) - - tmpl_data = self.get_template(report_xml) + tmpl_data = self.get_template() in_stream = StringIO(tmpl_data) out_stream = StringIO() template = Template(in_stream, out_stream, escape_false=True) - localcontext = parser_instance.localcontext + localcontext = self._get_parser_context(model_instance, data) if report_xml.py3o_is_local_fusion: template.render(localcontext) in_stream = out_stream @@ -181,7 +206,7 @@ def create_single_pdf(self, cr, uid, ids, data, report_xml, context=None): report_xml.py3o_server_id.url, data=fields, files=files) if r.status_code != 200: # server says we have an issue... let's tell that to enduser - raise exceptions.Warning( + raise UserError( _('Fusion server error %s') % r.text, ) @@ -189,38 +214,79 @@ def create_single_pdf(self, cr, uid, ids, data, report_xml, context=None): # we do nice chunked reading from the network... chunk_size = 1024 with NamedTemporaryFile( - suffix=filetype, - prefix='py3o-template-' + suffix=filetype, + prefix='py3o-template-' ) as fd: for chunk in r.iter_content(chunk_size): fd.write(chunk) fd.seek(0) # ... but odoo wants the whole data in memory anyways :) res = fd.read() + self._postprocess_report( + res, model_instance.id, save_in_attachment) + return res, "." + self.ir_actions_report_xml_id.py3o_filetype + + @api.multi + def _get_or_create_single_report(self, model_instance, data, + save_in_attachment): + self.ensure_one() + if save_in_attachment and save_in_attachment[ + 'loaded_documents'].get(model_instance.id): + d = save_in_attachment[ + 'loaded_documents'].get(model_instance.id) + return d, self.ir_actions_report_xml_id.py3o_filetype + return self._create_single_report( + model_instance, data, save_in_attachment) + + @api.multi + def _zip_results(self, results): + self.ensure_one() + zfname_prefix = self.ir_actions_report_xml_id.name + with NamedTemporaryFile(suffix="zip", prefix='py3o-zip-result') as fd: + with ZipFile(fd, 'w', ZIP_DEFLATED) as zf: + cpt = 0 + for r, ext in results: + fname = "%s_%d.%s" % (zfname_prefix, cpt, ext) + zf.writestr(fname, r) + cpt += 1 + fd.seek(0) + return fd.read(), 'zip' + + @api.multi + def _merge_pdfs(self, results): + from pyPdf import PdfFileWriter, PdfFileReader + output = PdfFileWriter() + for r in results: + reader = PdfFileReader(StringIO(r[0])) + for page in range(reader.getNumPages()): + output.addPage(reader.getPage(page)) + s = StringIO() + output.write(s) + return s.getvalue(), formats.FORMAT_PDF + + @api.multi + def _merge_results(self, results): + self.ensure_one() + if not results: + return False, False + if len(results) == 1: + return results[0] + filetype = self.ir_actions_report_xml_id.py3o_filetype + if filetype == formats.FORMAT_PDF: + return self._merge_pdfs(results) + else: + return self._zip_results(results) - return res, "." + filetype - - def create(self, cr, uid, ids, data, context=None): + @api.multi + def create_report(self, res_ids, data): """ Override this function to handle our py3o report """ - pool = registry(cr.dbname) - ir_action_report_obj = pool['ir.actions.report.xml'] - report_xml_ids = ir_action_report_obj.search( - cr, uid, [('report_name', '=', self.name[7:])], context=context - ) - if not report_xml_ids: - return super(Py3oParser, self).create( - cr, uid, ids, data, context=context - ) - - report_xml = ir_action_report_obj.browse( - cr, uid, report_xml_ids[0], context=context - ) - - result = self.create_source_pdf( - cr, uid, ids, data, report_xml, context - ) - - if not result: - return False, False - return result + model_instances = self.env[self.ir_actions_report_xml_id.model].browse( + res_ids) + save_in_attachment = self._check_attachment_use( + model_instances, self.ir_actions_report_xml_id) or {} + results = [] + for model_instance in model_instances: + results.append(self._get_or_create_single_report( + model_instance, data, save_in_attachment)) + return self._merge_results(results) diff --git a/report_py3o/tests/test_report_py3o.py b/report_py3o/tests/test_report_py3o.py index ac58f59cd8..5f273cb9ae 100644 --- a/report_py3o/tests/test_report_py3o.py +++ b/report_py3o/tests/test_report_py3o.py @@ -11,7 +11,7 @@ from openerp.tests.common import TransactionCase from openerp.exceptions import ValidationError -from ..py3o_parser import TemplateNotFound +from ..models.py3o_report import TemplateNotFound from base64 import b64encode @@ -56,9 +56,10 @@ def test_required_py3_filetype(self): "Field 'Output Format' is required for Py3O report") def test_reports(self): + py3o_report = self.env['py3o.report'] report = self.env.ref("report_py3o.res_users_report_py3o") - with mock.patch('openerp.addons.report_py3o.py3o_parser.' - 'Py3oParser.create_single_pdf') as patched_pdf: + with mock.patch.object( + py3o_report.__class__, '_create_single_report') as patched_pdf: # test the call the the create method inside our custom parser report.render_report(self.env.user.ids, report.report_name, @@ -98,7 +99,7 @@ def test_report_template_configs(self): report.render_report( self.env.user.ids, report.report_name, {}) - # the template can also be provivided as an abspaath + # the template can also be provided as an abspaath report.py3o_template_fallback = flbk_filename res = report.render_report( self.env.user.ids, report.report_name, {}) From 9befca91d45ac09416a4a1d680089bfa1c0c308c Mon Sep 17 00:00:00 2001 From: Laurent Mignon Date: Fri, 23 Dec 2016 12:43:57 +0100 Subject: [PATCH 5/6] [IMP] Minimizes memory consumption --- report_py3o/models/py3o_report.py | 182 +++++++++++++++----------- report_py3o/tests/test_report_py3o.py | 9 +- 2 files changed, 114 insertions(+), 77 deletions(-) diff --git a/report_py3o/models/py3o_report.py b/report_py3o/models/py3o_report.py index 9b57a703e0..33487ee1c0 100644 --- a/report_py3o/models/py3o_report.py +++ b/report_py3o/models/py3o_report.py @@ -8,12 +8,15 @@ import json import logging import os +from contextlib import closing + import pkg_resources import requests import sys -from tempfile import NamedTemporaryFile +import tempfile from zipfile import ZipFile, ZIP_DEFLATED +from openerp.exceptions import AccessError from openerp.exceptions import UserError from openerp.report.report_sxw import rml_parse from openerp import api, fields, models, _ @@ -149,18 +152,40 @@ def _get_parser_context(self, model_instance, data): self._extend_parser_context(context_instance, report_xml) return context_instance.localcontext - @api.multi - def _postprocess_report(self, content, res_id, save_in_attachment): + @api.model + def _get_report_from_name(self, report_name): + """Get the first record of ir.actions.report.xml having the + ``report_name`` as value for the field report_name. + """ + res = super(Py3oReport, self)._get_report_from_name(report_name) + if res: + return res + # maybe a py3o reprot + report_obj = self.env['ir.actions.report.xml'] + return report_obj.search( + [('report_type', '=', 'py3o'), + ('report_name', '=', report_name)]) + + @api.model + def _postprocess_report(self, report_path, res_id, save_in_attachment): if save_in_attachment.get(res_id): - attachment = { - 'name': save_in_attachment.get(res_id), - 'datas': base64.encodestring(content), - 'datas_fname': save_in_attachment.get(res_id), - 'res_model': save_in_attachment.get('model'), - 'res_id': res_id, - } - return self.env['ir.attachment'].create(attachment) - return False + with open(report_path, 'rb') as pdfreport: + attachment = { + 'name': save_in_attachment.get(res_id), + 'datas': base64.encodestring(pdfreport.read()), + 'datas_fname': save_in_attachment.get(res_id), + 'res_model': save_in_attachment.get('model'), + 'res_id': res_id, + } + try: + self.env['ir.attachment'].create(attachment) + except AccessError: + logger.info("Cannot save PDF report %r as attachment", + attachment['name']) + else: + logger.info( + 'The PDF document %s is now saved in the database', + attachment['name']) @api.multi def _create_single_report(self, model_instance, data, save_in_attachment): @@ -168,30 +193,31 @@ def _create_single_report(self, model_instance, data, save_in_attachment): """ self.ensure_one() report_xml = self.ir_actions_report_xml_id - + filetype = report_xml.py3o_filetype + result_fd, result_path = tempfile.mkstemp( + suffix='.' + filetype, prefix='p3o.report.tmp.') tmpl_data = self.get_template() in_stream = StringIO(tmpl_data) - out_stream = StringIO() - template = Template(in_stream, out_stream, escape_false=True) - localcontext = self._get_parser_context(model_instance, data) - if report_xml.py3o_is_local_fusion: - template.render(localcontext) - in_stream = out_stream - datadict = {} - else: - expressions = template.get_all_user_python_expression() - py_expression = template.convert_py3o_to_python_ast(expressions) - convertor = Py3oConvertor() - data_struct = convertor(py_expression) - datadict = data_struct.render(localcontext) - - filetype = report_xml.py3o_filetype - is_native = Formats().get_format(filetype).native - if is_native: - res = out_stream.getvalue() - else: # Call py3o.server to render the template in the desired format - in_stream.seek(0) + with closing(os.fdopen(result_fd, 'w+')) as out_stream: + template = Template(in_stream, out_stream, escape_false=True) + localcontext = self._get_parser_context(model_instance, data) + is_native = Formats().get_format(filetype).native + if report_xml.py3o_is_local_fusion: + template.render(localcontext) + out_stream.seek(0) + in_stream = out_stream.read() + datadict = {} + else: + expressions = template.get_all_user_python_expression() + py_expression = template.convert_py3o_to_python_ast( + expressions) + convertor = Py3oConvertor() + data_struct = convertor(py_expression) + datadict = data_struct.render(localcontext) + + if not is_native or not report_xml.py3o_is_local_fusion: + # Call py3o.server to render the template in the desired format files = { 'tmpl_file': in_stream, } @@ -210,21 +236,13 @@ def _create_single_report(self, model_instance, data, save_in_attachment): _('Fusion server error %s') % r.text, ) - # Here is a little joke about Odoo - # we do nice chunked reading from the network... chunk_size = 1024 - with NamedTemporaryFile( - suffix=filetype, - prefix='py3o-template-' - ) as fd: + with open(result_path, 'w+') as fd: for chunk in r.iter_content(chunk_size): fd.write(chunk) - fd.seek(0) - # ... but odoo wants the whole data in memory anyways :) - res = fd.read() self._postprocess_report( - res, model_instance.id, save_in_attachment) - return res, "." + self.ir_actions_report_xml_id.py3o_filetype + result_path, model_instance.id, save_in_attachment) + return result_path @api.multi def _get_or_create_single_report(self, model_instance, data, @@ -239,43 +257,42 @@ def _get_or_create_single_report(self, model_instance, data, model_instance, data, save_in_attachment) @api.multi - def _zip_results(self, results): + def _zip_results(self, reports_path): self.ensure_one() zfname_prefix = self.ir_actions_report_xml_id.name - with NamedTemporaryFile(suffix="zip", prefix='py3o-zip-result') as fd: - with ZipFile(fd, 'w', ZIP_DEFLATED) as zf: - cpt = 0 - for r, ext in results: - fname = "%s_%d.%s" % (zfname_prefix, cpt, ext) - zf.writestr(fname, r) - cpt += 1 - fd.seek(0) - return fd.read(), 'zip' + result_path = tempfile.mktemp(suffix="zip", prefix='py3o-zip-result') + with ZipFile(result_path, 'w', ZIP_DEFLATED) as zf: + cpt = 0 + for report in reports_path: + fname = "%s_%d.%s" % ( + zfname_prefix, cpt, report.split('.')[-1]) + zf.write(report, fname) - @api.multi - def _merge_pdfs(self, results): - from pyPdf import PdfFileWriter, PdfFileReader - output = PdfFileWriter() - for r in results: - reader = PdfFileReader(StringIO(r[0])) - for page in range(reader.getNumPages()): - output.addPage(reader.getPage(page)) - s = StringIO() - output.write(s) - return s.getvalue(), formats.FORMAT_PDF + cpt += 1 + return result_path @api.multi - def _merge_results(self, results): + def _merge_results(self, reports_path): self.ensure_one() - if not results: - return False, False - if len(results) == 1: - return results[0] filetype = self.ir_actions_report_xml_id.py3o_filetype + if not reports_path: + return False, False + if len(reports_path) == 1: + return reports_path[0], filetype if filetype == formats.FORMAT_PDF: - return self._merge_pdfs(results) + return self._merge_pdf(reports_path), formats.FORMAT_PDF else: - return self._zip_results(results) + return self._zip_results(reports_path), 'zip' + + @api.model + def _cleanup_tempfiles(self, temporary_files): + # Manual cleanup of the temporary files + for temporary_file in temporary_files: + try: + os.unlink(temporary_file) + except (OSError, IOError): + logger.error( + 'Error when trying to remove file %s' % temporary_file) @api.multi def create_report(self, res_ids, data): @@ -285,8 +302,21 @@ def create_report(self, res_ids, data): res_ids) save_in_attachment = self._check_attachment_use( model_instances, self.ir_actions_report_xml_id) or {} - results = [] + reports_path = [] for model_instance in model_instances: - results.append(self._get_or_create_single_report( - model_instance, data, save_in_attachment)) - return self._merge_results(results) + reports_path.append( + self._get_or_create_single_report( + model_instance, data, save_in_attachment)) + + result_path, filetype = self._merge_results(reports_path) + reports_path.append(result_path) + + # Here is a little joke about Odoo + # we do all the generation process using files to avoid memory + # consumption... + # ... but odoo wants the whole data in memory anyways :) + + with open(result_path, 'r+b') as fd: + res = fd.read() + self._cleanup_tempfiles(set(reports_path)) + return res, filetype diff --git a/report_py3o/tests/test_report_py3o.py b/report_py3o/tests/test_report_py3o.py index 5f273cb9ae..56bc385ddb 100644 --- a/report_py3o/tests/test_report_py3o.py +++ b/report_py3o/tests/test_report_py3o.py @@ -5,6 +5,7 @@ import mock import os import pkg_resources +import tempfile from py3o.formats import Formats @@ -60,11 +61,17 @@ def test_reports(self): report = self.env.ref("report_py3o.res_users_report_py3o") with mock.patch.object( py3o_report.__class__, '_create_single_report') as patched_pdf: + result = tempfile.mktemp('.txt') + with open(result, 'w') as fp: + fp.write('dummy') + patched_pdf.return_value = result # test the call the the create method inside our custom parser report.render_report(self.env.user.ids, report.report_name, {}) self.assertEqual(1, patched_pdf.call_count) + # generated files no more exists + self.assertFalse(os.path.exists(result)) res = report.render_report( self.env.user.ids, report.report_name, {}) self.assertTrue(res) @@ -79,7 +86,7 @@ def test_reports(self): magick_response.iter_content.return_value = "test result" res = report.render_report( self.env.user.ids, report.report_name, {}) - self.assertEqual(('test result', '.pdf'), res) + self.assertEqual(('test result', 'pdf'), res) def test_report_template_configs(self): report = self.env.ref("report_py3o.res_users_report_py3o") From e662a1a504636a0c5db89a58a20dc0be14765cc9 Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Wed, 25 Jan 2017 11:56:21 +0100 Subject: [PATCH 6/6] [IMP] Allow to override/extend the way we get the fallback template --- report_py3o/models/py3o_report.py | 58 ++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/report_py3o/models/py3o_report.py b/report_py3o/models/py3o_report.py index 33487ee1c0..aef2edc2d3 100644 --- a/report_py3o/models/py3o_report.py +++ b/report_py3o/models/py3o_report.py @@ -82,7 +82,41 @@ class Py3oReport(models.TransientModel): ) @api.multi - def get_template(self): + def _get_template_from_path(self, tmpl_name): + """"Return the template from the path to root of the module if specied + or an absolute path on your server + """ + if not tmpl_name: + return None + report_xml = self.ir_actions_report_xml_id + flbk_filename = None + if report_xml.module: + # if the default is defined + flbk_filename = pkg_resources.resource_filename( + "openerp.addons.%s" % report_xml.module, + tmpl_name, + ) + elif os.path.isabs(tmpl_name): + # It is an absolute path + flbk_filename = os.path.normcase(os.path.normpath(tmpl_name)) + if flbk_filename and os.path.exists(flbk_filename): + # and it exists on the fileystem + with open(flbk_filename, 'r') as tmpl: + return tmpl.read() + return None + + @api.multi + def _get_template_fallback(self, model_instance): + """ + Return the template referenced in the report definition + :return: + """ + self.ensure_one() + report_xml = self.ir_actions_report_xml_id + return self._get_template_from_path(report_xml.py3o_template_fallback) + + @api.multi + def get_template(self, model_instance): """private helper to fetch the template data either from the database or from the default template file provided by the implementer. @@ -96,30 +130,14 @@ def get_template(self): openerp.exceptions.DeferredException """ self.ensure_one() - tmpl_data = None report_xml = self.ir_actions_report_xml_id if report_xml.py3o_template_id and report_xml.py3o_template_id.id: # if a user gave a report template tmpl_data = b64decode( report_xml.py3o_template_id.py3o_template_data ) - - elif report_xml.py3o_template_fallback: - tmpl_name = report_xml.py3o_template_fallback - flbk_filename = None - if report_xml.module: - # if the default is defined - flbk_filename = pkg_resources.resource_filename( - "openerp.addons.%s" % report_xml.module, - tmpl_name, - ) - elif os.path.isabs(tmpl_name): - # It is an absolute path - flbk_filename = os.path.normcase(os.path.normpath(tmpl_name)) - if flbk_filename and os.path.exists(flbk_filename): - # and it exists on the fileystem - with open(flbk_filename, 'r') as tmpl: - tmpl_data = tmpl.read() + else: + tmpl_data = self._get_template_fallback(model_instance) if tmpl_data is None: # if for any reason the template is not found @@ -196,7 +214,7 @@ def _create_single_report(self, model_instance, data, save_in_attachment): filetype = report_xml.py3o_filetype result_fd, result_path = tempfile.mkstemp( suffix='.' + filetype, prefix='p3o.report.tmp.') - tmpl_data = self.get_template() + tmpl_data = self.get_template(model_instance) in_stream = StringIO(tmpl_data) with closing(os.fdopen(result_fd, 'w+')) as out_stream: