From 4169e568a7fde0032be63991dca9e5e584b1eede Mon Sep 17 00:00:00 2001 From: Sagar Date: Mon, 3 Jul 2017 15:02:33 +0530 Subject: [PATCH 1/3] PR for Billing Report --- cloud/endagaweb/models.py | 1 + .../js/dashboard/report-chart-components.js | 849 ++++++++++++++++++ cloud/endagaweb/stats_app/stats_client.py | 96 +- cloud/endagaweb/stats_app/views.py | 35 +- .../endagaweb/templates/dashboard/layout.html | 1 + .../templates/dashboard/report/billing.html | 368 ++++++++ .../templates/dashboard/report/filter.html | 75 ++ .../templates/dashboard/report/header.html | 19 + .../templates/dashboard/report/nav.html | 19 + cloud/endagaweb/tests/test_reports_ui.py | 79 ++ cloud/endagaweb/urls.py | 5 +- cloud/endagaweb/views/__init__.py | 1 + cloud/endagaweb/views/reports.py | 220 +++++ 13 files changed, 1762 insertions(+), 6 deletions(-) create mode 100644 cloud/endagaweb/static/js/dashboard/report-chart-components.js create mode 100644 cloud/endagaweb/templates/dashboard/report/billing.html create mode 100644 cloud/endagaweb/templates/dashboard/report/filter.html create mode 100644 cloud/endagaweb/templates/dashboard/report/header.html create mode 100644 cloud/endagaweb/templates/dashboard/report/nav.html create mode 100644 cloud/endagaweb/tests/test_reports_ui.py create mode 100644 cloud/endagaweb/views/reports.py diff --git a/cloud/endagaweb/models.py b/cloud/endagaweb/models.py index 65b0780f..2cc602a1 100644 --- a/cloud/endagaweb/models.py +++ b/cloud/endagaweb/models.py @@ -525,6 +525,7 @@ class Subscriber(models.Model): # When toggled, this will protect a subsriber from getting "vacuumed." You # can still delete subs with the usual "deactivate" button. prevent_automatic_deactivation = models.BooleanField(default=False) + role = models.TextField(default='retailer') @classmethod def update_balance(cls, imsi, other_bal): diff --git a/cloud/endagaweb/static/js/dashboard/report-chart-components.js b/cloud/endagaweb/static/js/dashboard/report-chart-components.js new file mode 100644 index 00000000..390c6efc --- /dev/null +++ b/cloud/endagaweb/static/js/dashboard/report-chart-components.js @@ -0,0 +1,849 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +// React chart components. + + +var TimeseriesChartWithButtonsAndDatePickers = React.createClass({ + getInitialState: function() { + // We expect many of these values to be overridden before the chart is + // first rendered -- see componentDidMount. + return { + startTimeEpoch: 0, + endTimeEpoch: -1, + isLoading: true, + chartData: {}, + activeButtonText: '', + xAxisFormatter: '%x', + yAxisFormatter: '', + activeView:'' + } + }, + getDefaultProps: function() { + // Load the current time with the user's clock if nothing is specified. We + // also specify the user's computer's timezone offset and use that to + // adjust the graph data. + var currentTime = Math.round(new Date().getTime() / 1000); + return { + title: 'title (set me!)', + chartID: 'one', + buttons: ['hour', 'day', 'week', 'month', 'year'], + icons: ['graph', 'list'], + defaultView: 'graph', + defaultButtonText: 'week', + endpoint: '/api/v1/stats/', + statTypes: 'sms', + level: 'network', + levelID: 0, + aggregation: 'count', + yAxisLabel: 'an axis label (set me!)', + currentTimeEpoch: currentTime, + timezoneOffset: 0, + tooltipUnits: '', + frontTooltip: '', + chartType: 'line-chart', + reportView: 'list' + } + }, + + // On-mount, build the charts with the default data. + // Note that we have to load the current time here. + componentDidMount: function() { + this.setState({ + activeButtonText: this.props.defaultButtonText, + activeView: this.props.defaultView, + startTimeEpoch: this.props.currentTimeEpoch - secondsMap[this.props.defaultButtonText], + endTimeEpoch: this.props.currentTimeEpoch, + // When the request params in the state have been set, go get more data. + }, function() { + this.updateChartData(); + }); + }, + + componentDidUpdate(prevProps, prevState) { + // Update if we toggled a load + if (!prevState.isLoading && this.state.isLoading) { + this.updateChartData(); + } + }, + + // This handler takes the text of the date range buttons + // and ouputs figures out the corresponding number of seconds. + handleButtonClick: function(text) { + // Update only if the startTime has actually changed. + var newStartTimeEpoch = this.props.currentTimeEpoch - secondsMap[text]; + if (this.state.startTimeEpoch != newStartTimeEpoch) { + this.setState({ + startTimeEpoch: newStartTimeEpoch, + endTimeEpoch: this.props.currentTimeEpoch, + isLoading: true, + activeButtonText: text, + }); + } + }, + + // This handler takes the text of the view mode buttons + // and ouputs figures out the corresponding number of seconds. + handleViewClick: function(text) { + // Update only if the startTime has actually changed. + if (this.state.activeView != text) { + + this.setState({ + startTimeEpoch: this.state.startTimeEpoch, + endTimeEpoch: this.props.currentTimeEpoch, + isLoading: true, + activeView: text, + }); + var interval = this.props.defaultButtonText; + setTimeout(function(){ + //this.handleButtonClick(interval); + this.forceUpdate() + }.bind(this), 1000); + + } + }, + + handleDownloadClick: function(text) { + var queryParams = { + 'start-time-epoch': this.state.startTimeEpoch, + 'end-time-epoch': this.state.endTimeEpoch, + 'stat-types': this.props.statTypes, + 'level':this.props.level, + 'level_id': this.props.levelID, + 'report-type':this.props.title, + + }; + $.get('/report/downloadcsv', queryParams, function(data,response) { + + var todayTime = new Date(); var month = (todayTime .getMonth() + 1); var day = (todayTime .getDate()); var year = (todayTime .getFullYear()); + var convertdate = year+ "-" + month + "-" +day; + this.setState({ + isLoading: false, + data: data, + title:this.props.title + }); + var filename = this.state.title + var csvData = new Blob([data], {type: 'text/csv;charset=utf-8;'}); + var csvURL = null; + if (navigator.msSaveBlob) { + csvURL = navigator.msSaveBlob(csvData, filename+"-"+convertdate+'.csv'); + } else { + csvURL = window.URL.createObjectURL(csvData); + } + var tempLink = document.createElement('a'); + document.body.appendChild(tempLink); + tempLink.href = csvURL; + tempLink.setAttribute('download', filename+"-"+convertdate+'.csv'); + tempLink.target="_self" + tempLink.click(); + }.bind(this)); + }, + + // Datepicker handlers, one each for changing the start and end times. + startTimeChange: function(newTime) { + if (newTime < this.state.endTimeEpoch && !this.state.isLoading) { + this.setState({ + startTimeEpoch: newTime, + isLoading: true, + }); + } + }, + endTimeChange: function(newTime) { + var now = moment().unix(); + if (newTime > this.state.startTimeEpoch && newTime < now + && !this.state.isLoading) { + this.setState({ + endTimeEpoch: newTime, + isLoading: true, + }); + } + }, + + // Gets new chart data from the backend. + // Recall that this must be called explicitly..it's a bit different + // than the normal react component connectivity. + // First figure out how to set the interval (or, more aptly, the bin width) + // and the x-axis formatter (see the d3 wiki on time formatting). + updateChartData: function() { + var interval, newXAxisFormatter, newYAxisFormatter; + var delta = this.state.endTimeEpoch - this.state.startTimeEpoch; + if (delta <= 60) { + interval = 'seconds'; + newXAxisFormatter = '%-H:%M:%S'; + } else if (delta <= (60 * 60)) { + interval = 'minutes'; + newXAxisFormatter = '%-H:%M'; + } else if (delta <= (24 * 60 * 60)) { + interval = 'hours'; + newXAxisFormatter = '%-H:%M'; + } else if (delta <= (7 * 24 * 60 * 60)) { + interval = 'hours'; + newXAxisFormatter = '%b %d, %-H:%M'; + } else if (delta <= (30 * 24 * 60 * 60)) { + interval = 'days'; + newXAxisFormatter = '%b %d'; + } else if (delta <= (365 * 24 * 60 * 60)) { + interval = 'days'; + newXAxisFormatter = '%b %d'; + } else { + interval = 'months'; + newXAxisFormatter = '%x'; + } + if (this.props.statTypes == 'total_data,uploaded_data,downloaded_data') { + newYAxisFormatter = '.1f'; + } else { + newYAxisFormatter = ''; + } + var queryParams = { + 'start-time-epoch': this.state.startTimeEpoch, + 'end-time-epoch': this.state.endTimeEpoch, + 'interval': interval, + 'stat-types': this.props.statTypes, + 'level-id': this.props.levelID, + 'aggregation': this.props.aggregation, + 'extras': this.props.extras, + 'dynamic-stat': this.props.dynamicStat, + 'topup-percent': this.props.topupPercent, + 'report-view': this.props.reportView + }; + var endpoint = this.props.endpoint + this.props.level; + $.get(endpoint, queryParams, function(data) { + this.setState({ + isLoading: false, + chartData: data, + xAxisFormatter: newXAxisFormatter, + yAxisFormatter: newYAxisFormatter, + }); + }.bind(this)); + }, + + render: function() { + var fromDatepickerID = 'from-datepicker-' + this.props.chartID; + var toDatepickerID = 'to-datepicker-' + this.props.chartID; + return ( +
+

{this.props.title}

+ past    + {this.props.buttons.map(function(buttonText, index) { + return ( + + ); + }, this)} + + + + +     + {this.props.icons.map(function(buttonText, index) { + return ( + + ); + }, this)} + + + + + +
+ ); + }, +}); + + +var secondsMap = { + 'hour': 60 * 60, + 'day': 24 * 60 * 60, + 'week': 7 * 24 * 60 * 60, + 'month': 30 * 24 * 60 * 60, + 'year': 365 * 24 * 60 * 60, +}; + +var add = function (a, b){ + return a + b ; +} + +// Builds the target chart from scratch. NVD3 surprisingly handles this well. +// domTarget is the SVG element's parent and data is the info that will be graphed. +var updateChart = function(domTarget, data, xAxisFormatter, yAxisFormatter, yAxisLabel, timezoneOffset, tooltipUnits, frontTooltip, chartType, domTargetId) { + // We pass in the timezone offset and calculate a locale offset. The former + // is based on the UserProfile's specified timezone and the latter is the user's + // computer's timezone offset. We manually shift the data to work around + // d3's conversion into the user's computer's timezone. Note that for my laptop + // in PST, the locale offset is positive (7hrs) while the UserProfile offset + // is negative (-7hrs). + var localeOffset = 60 * (new Date()).getTimezoneOffset(); + var shiftedData = []; + var changeAmount = []; + var tableData = []; + + for (var index in data) { + var newSeries = { 'key': data[index]['key'] }; + var newValues = []; + if( typeof(data[index]['values']) === 'object'){ + for (var series_index in data[index]['values']) { + var newValue = [ + // Shift out of the locale offset to 'convert' to UTC and then shift + // back into the operator's tz by adding the tz offset from the server. + data[index]['values'][series_index][0] + 1e3 * localeOffset + 1e3 * timezoneOffset, + data[index]['values'][series_index][1] + ]; + newValues.push(newValue); + changeAmount.push(newValue[1]) + } + // Get sum of the total charges + var sumAmount = changeAmount.reduce(add, 0); + changeAmount = [] + // sum can be of all negative values + if ( sumAmount < 0 ){ + newSeries['total'] = (sumAmount * -1); + } + else{ + newSeries['total'] = (sumAmount); + } + newSeries['values'] = newValues; + } else { + newSeries['total'] = data[index]['values']; + } + tableData.push([newSeries['key'], newSeries['total']]); + shiftedData.push(newSeries); + } + + $('.'+domTargetId).DataTable({ + data: tableData, + paging: false, + ordering: false, + info: false, + searching: false, + autoWidth: true, + scrollY: 320, + destroy: true, + columns: [ + { title: "Title" }, + { title: "Value" } + ] + }); + + + nv.addGraph(function() { + if(chartType == 'pie-chart') { + var chart = nv.models.pieChart() + .x(function(d) { return d.key.replace('_'," "); }) + .y(function(d) { return d.total; }) + .showLabels(true) + .labelType("percent"); + + chart.tooltipContent(function(key, x, y) { + return '

'+ key + '

' + '
'+ '' + '

' + frontTooltip + x + '

'+ '' + '

' + }); + + d3.select(domTarget) + .datum(shiftedData) + .transition().duration(1200) + .call(chart); + } else if(chartType == 'bar-chart'){ + var chart = nv.models.multiBarChart() + .x(function(d) { return d[0] }) + .y(function(d) { return d[1] }) + //.staggerLabels(true) //Too many bars and not enough room? Try staggering labels. + // .tooltips(true) + // .showValues(true) //...instead, show the bar value right on top of each bar. + .transitionDuration(350) + .stacked(false).showControls(false); + + chart.xAxis.tickFormat(function(d) { + return d3.time.format(xAxisFormatter)(new Date(d)); + }); + // Fixes x-axis time alignment. + + var xScale =d3.time.scale.utc(); + + chart.yAxis.scale(xScale) + .axisLabel(yAxisLabel) + .axisLabelDistance(25) + .tickFormat(d3.format(yAxisFormatter)); + // Fixes the axis-labels being rendered out of the SVG element. + chart.margin({right: 80}); + chart.tooltipContent(function(key, x, y) { + return '

' + frontTooltip + y + tooltipUnits + ' ' + key + '

' + '

' + x + '

'; + }); + + + d3.select(domTarget) + .datum(shiftedData) + .transition().duration(350) + .call(chart); + } else { + var chart = nv.models.lineChart() + .x(function(d) { return d[0] }) + .y(function(d) { return d[1] }) + .color(d3.scale.category10().range()) + .interpolate('monotone') + .showYAxis(true) + ; + chart.xAxis + .tickFormat(function(d) { + return d3.time.format(xAxisFormatter)(new Date(d)); + }); + // Fixes x-axis time alignment. + chart.xScale(d3.time.scale.utc()); + chart.yAxis + .axisLabel(yAxisLabel) + .axisLabelDistance(25) + .tickFormat(d3.format(yAxisFormatter)); + // Fixes the axis-labels being rendered out of the SVG element. + chart.margin({right: 80}); + chart.tooltipContent(function(key, x, y) { + return '

' + frontTooltip + y + tooltipUnits + ' ' + key + '

' + '

' + x + '

'; + }); + + d3.select(domTarget) + .datum(shiftedData) + .transition().duration(350) + .call(chart); + } + // Resize the chart on window resize. + nv.utils.windowResize(chart.update); + return chart; + }); +}; + + +var TimeseriesChart = React.createClass({ + + getDefaultProps: function() { + return { + chartID: 'some-chart-id', + chartHeight: 380, + data: {}, + xAxisFormatter: '%x', + yAxisFormatter: '', + yAxisLabel: 'the y axis!', + timezoneOffset: 0, + frontTooltip: '', + tooltipUnits: '', + chartType:'', + activeView:'' + } + }, + + chartIsFlat(results) { + return results.every(function(series) { + if(typeof(series['values']) === 'object'){ + return series['values'].every(function(pair) { + return pair[1] === 0; + }); + } else { + return series['values'] === 0; + } + }); + }, + + render: function() { + + var results = this.props.data['results']; + var isFlatChart = !results || this.chartIsFlat(results); + var className = ['time-series-chart-container']; + var flatLineOverlay = null; + if (isFlatChart) { + flatLineOverlay = ( +
+
+ No data available for this range. +
+
+ ); + className.push('flat'); + } + if(this.props.activeView == 'list') { + $('#'+this.props.chartID + "-download").hide(); + return ( +
+ {flatLineOverlay} + + + ); + } + else { + $('#'+this.props.chartID + "-download").show(); + return ( +
+ {flatLineOverlay} + +
+ ); + } + } +}); + + +var TimeSeriesChartElement = React.createClass({ + // When the request params have changed, get new data and rebuild the graph. + // We circumvent react's typical re-render cycle for this component by returning false. + shouldComponentUpdate: function(nextProps) { + this.props.activeView = nextProps.activeView; + var nextData = JSON.stringify(nextProps.data); + var prevData = JSON.stringify(this.props.data); + if (nextData !== prevData) { + updateChart( + '#' + this.props.chartID, + nextProps.data['results'], + nextProps.xAxisFormatter, + nextProps.yAxisFormatter, + nextProps.yAxisLabel, + this.props.timezoneOffset, + this.props.tooltipUnits, + this.props.frontTooltip, + this.props.chartType, + this.props.chartID + ); + } + return false; + }, + + render: function() { + var inlineStyles = { + height: this.props.chartHeight + }; + return ( + + + ); + } +}); + + +var RangeButton = React.createClass({ + propTypes: { + buttonText: React.PropTypes.string.isRequired + }, + + getDefaultProps: function() { + return { + buttonText: 'day', + activeButtonText: '', + onButtonClick: null, + } + }, + + render: function() { + // Determine whether this particular button is active by checking + // this button's text vs the owner's knowledge of the active button. + // Then change styles accordingly. + var inlineStyles = { + marginRight: 20 + }; + if (this.props.buttonText == this.props.activeButtonText) { + inlineStyles.cursor = 'inherit'; + inlineStyles.color = 'black'; + inlineStyles.textDecoration = 'none'; + } else { + inlineStyles.cursor = 'pointer'; + } + return ( + + {this.props.buttonText} + + ); + }, + + onThisClick: function(text) { + this.props.onButtonClick(text); + }, +}); + + +var LoadingText = React.createClass({ + getDefaultProps: function() { + return { + visible: false, + } + }, + + render: function() { + var inlineStyles = { + display: this.props.visible ? 'inline' : 'none', + marginRight: 20, + }; + + return ( + + (loading..) + + ); + }, +}); + + +var DatePicker = React.createClass({ + getDefaultProps: function() { + return { + label: 'date', + pickerID: 'some-datetimepicker-id', + epochTime: 0, + onDatePickerChange: null, + datePickerOptions : { + icons: { + time: 'fa fa-clock-o', + date: 'fa fa-calendar', + up: 'fa fa-arrow-up', + down: 'fa fa-arrow-down', + previous: 'fa fa-arrow-left', + next: 'fa fa-arrow-right', + today: 'fa fa-circle-o', + }, + showTodayButton: true, + format: 'YYYY-MM-DD [at] h:mmA', + }, + dateFormat: 'YYYY-MM-DD [at] h:mmA', + } + }, + + componentDidMount: function() { + var formattedDate = moment.unix(this.props.epochTime).format(this.props.dateFormat); + var domTarget = '#' + this.props.pickerID; + $(domTarget).keydown(function(e){ + e.preventDefault(); + return false; + }); + $(domTarget) + .datetimepicker(this.props.datePickerOptions) + .data('DateTimePicker') + .date(formattedDate); + var dateFormat = this.props.dateFormat; + var handler = this.props.onDatePickerChange; + $(domTarget).on('dp.change', function(event) { + var newEpochTime = moment(event.target.value, dateFormat).unix(); + handler(newEpochTime); + }); + }, + + shouldComponentUpdate: function(nextProps) { + var formattedDate = moment.unix(nextProps.epochTime).format(nextProps.dateFormat); + var domTarget = '#' + nextProps.pickerID; + $(domTarget).data('DateTimePicker').date(formattedDate); + return false + }, + + render: function() { + return ( + + + + + ); + }, +}); + +var DownloadButton = React.createClass({ + getDefaultProps: function() { + return { + visible: false, + startimeepoch:'', + endTimeEpoch:'', + defaultButtonText: 'week', + statsType:'', + onButtonClick: null, + activeView:'graph' + } + }, + componentWillMount() { + this.id = this.props.chartID + "-download"; + }, + componentDidMount: function() { + var domTargetId = this.props.chartID; + var btn = document.getElementById(this.id); + var svg = document.getElementById(domTargetId); + var canvas = document.querySelector('canvas'); + + btn.addEventListener('click', function () { + + var width = $("#"+domTargetId).width(); + var height = $("#"+domTargetId).height(); + + var canvas = document.getElementById('canvas'); + canvas.width = width; + canvas.height = height; + var ctx = canvas.getContext('2d'); + + ctx.fillStyle = "#FFF"; + ctx.fillRect(0, 0, width, height); + + var data = (new XMLSerializer()).serializeToString(svg); + var DOMURL = window.URL || window.webkitURL; + + var img = new Image(); + var svgBlob = new Blob([data], {type: 'image/svg+xml;charset=utf-8'}); + var url = DOMURL.createObjectURL(svgBlob); + + img.onload = function() { + ctx.drawImage(img, 0, 0); + DOMURL.revokeObjectURL(url); + + var imgURI = canvas + .toDataURL('image/png') + .replace('image/png', 'image/octet-stream'); + + var evt = new MouseEvent('click', { + view: window, + bubbles: false, + cancelable: true + }); + + var a = document.createElement('a'); + a.setAttribute('download', 'report.png'); + a.setAttribute('href', imgURI); + a.setAttribute('target', '_blank'); + a.dispatchEvent(evt); + }; + img.src = url; + //img.setAttribute("src", "data:image/svg+xml;base64," + btoa(unescape(encodeURIComponent(data))) ); + }); + }, + render: function() { + return ( + + + +    + + + + + ); + }, + onThisClick: function(text) { + this.props.onButtonClick(); + } +}); + +var ViewButton = React.createClass({ + propTypes: { + buttonText: React.PropTypes.string.isRequired + }, + + getDefaultProps: function() { + return { + buttonText: 'graph', + activeView: '', + onButtonClick: null, + } + }, + + render: function() { + // Determine whether this particular button is active by checking + // this button's text vs the owner's knowledge of the active button. + // Then change styles accordingly. + var inlineStyles = { + marginRight: 20 + }; + if (this.props.buttonText == this.props.activeView) { + inlineStyles.cursor = 'inherit'; + inlineStyles.color = 'black'; + inlineStyles.textDecoration = 'none'; + } else { + inlineStyles.cursor = 'pointer'; + } + if(this.props.buttonText == 'graph') { + return ( + + + + ); + } else { + return ( + + + + ); + } + }, + + onThisClick: function(text) { + this.props.onButtonClick(text); + }, +}); + +var Table = React.createClass({ + shouldComponentUpdate: function(nextProps) { + this.props.activeView = nextProps.activeView; + + var nextData = JSON.stringify(nextProps.data); + var prevData = JSON.stringify(this.props.data); + if (nextData !== prevData) { + updateChart( + '#' + this.props.chartID, + nextProps.data['results'], + nextProps.xAxisFormatter, + nextProps.yAxisFormatter, + nextProps.yAxisLabel, + this.props.timezoneOffset, + this.props.tooltipUnits, + this.props.frontTooltip, + this.props.chartType, + this.props.chartID + ); + } + return false; + }, + + render: function() { + var inlineStyles = { + 'min-height': '360px', + 'margin-top': '20px' + }; + return ( +
+
+
+ ); + } +}); diff --git a/cloud/endagaweb/stats_app/stats_client.py b/cloud/endagaweb/stats_app/stats_client.py index 5ebe043f..2f6c41f1 100644 --- a/cloud/endagaweb/stats_app/stats_client.py +++ b/cloud/endagaweb/stats_app/stats_client.py @@ -17,6 +17,8 @@ import qsstats from endagaweb import models +from operator import itemgetter + CALL_KINDS = [ @@ -25,7 +27,9 @@ SMS_KINDS = [ 'local_sms', 'local_recv_sms', 'outside_sms', 'incoming_sms', 'free_sms', 'error_sms'] -USAGE_EVENT_KINDS = CALL_KINDS + SMS_KINDS + ['gprs'] +TRANSFER_KINDS = ['transfer', 'add-money'] +DENOMINATION_KINDS = ['start_amount', 'end_amount'] +USAGE_EVENT_KINDS = CALL_KINDS + SMS_KINDS + ['gprs'] + TRANSFER_KINDS TIMESERIES_STAT_KEYS = [ 'ccch_sdcch4_load', 'tch_f_max', 'tch_f_load', 'sdcch8_max', 'tch_f_pdch_load', 'tch_f_pdch_max', 'tch_h_load', 'tch_h_max', 'sdcch8_load', 'ccch_sdcch4_max', 'sdcch_load', 'sdcch_available', 'tchf_load', 'tchf_available', @@ -99,6 +103,7 @@ def aggregate_timeseries(self, param, **kwargs): end_time_epoch = kwargs.pop('end_time_epoch', -1) interval = kwargs.pop('interval', 'months') aggregation = kwargs.pop('aggregation', 'count') + report_view = kwargs.pop('report_view', 'list') # Turn the start and end epoch timestamps into datetimes. start = datetime.fromtimestamp(start_time_epoch, pytz.utc) if end_time_epoch != -1: @@ -114,6 +119,10 @@ def aggregate_timeseries(self, param, **kwargs): elif param in TIMESERIES_STAT_KEYS: objects = models.TimeseriesStat.objects filters = Q(key=param) + else: + # For Dynamic Kinds coming from database currently for Top Up + objects = models.UsageEvent.objects + filters = Q(kind='transfer') # Filter by infrastructure level. if self.level == 'tower': filters = filters & Q(bts__id=self.level_id) @@ -121,6 +130,13 @@ def aggregate_timeseries(self, param, **kwargs): filters = filters & Q(network__id=self.level_id) elif self.level == 'global': pass + if kwargs.has_key('query'): + filters = filters & kwargs.pop('query') + if report_view == 'value': + filters = filters & Q(date__lte=end) & Q(date__gte=start) + result = models.UsageEvent.objects.filter(filters).values_list( + 'subscriber_id', flat=True).distinct() + return list(result) # Create the queryset itself. queryset = objects.filter(filters) # Use qsstats to aggregate the queryset data on an interval. @@ -136,6 +152,43 @@ def aggregate_timeseries(self, param, **kwargs): elif aggregation == 'average_value': queryset_stats = qsstats.QuerySetStats( queryset, 'date', aggregate=aggregates.Avg('value')) + # Sum of change in amounts for SMS/CALL + elif aggregation in ['transaction_sum', 'transcation_count']: + queryset_stats = qsstats.QuerySetStats( + # Change is negative value, set positive for charts + queryset, 'date', aggregate=( + aggregates.Sum('change') * -1)) + # if percentage is set for top top-up + percentage = kwargs['topup_percent'] + top_numbers = 1 + if percentage is not None: + numbers = {} + percentage = float(percentage) / 100 + # Create subscribers dict + for query in queryset: + numbers[query.to_number] = 0 + for query in queryset_stats.qs: + numbers[query.to_number] += (query.change * -1) + top_numbers = int(len(numbers) * percentage) + top_numbers = top_numbers if top_numbers > 1 else top_numbers + top_subscribers = list(dict( + sorted(numbers.iteritems(), key=itemgetter(1), + reverse=True)[:top_numbers]).keys()) + queryset = queryset_stats.qs.filter( + Q(to_number__in=top_subscribers)) + # Count the numbers + if aggregation == 'transcation_count': + queryset_stats = qsstats.QuerySetStats( + queryset, 'date', aggregate=( + aggregates.Count('to_number'))) + else: + # Sum of change + queryset_stats = qsstats.QuerySetStats( + queryset, 'date', aggregate=( + aggregates.Sum('change') * -1)) + elif aggregation == 'loader': + queryset_stats = qsstats.QuerySetStats( + queryset, 'date', aggregate=aggregates.Count('subscriber_id')) else: queryset_stats = qsstats.QuerySetStats(queryset, 'date') timeseries = queryset_stats.time_series(start, end, interval=interval) @@ -143,6 +196,16 @@ def aggregate_timeseries(self, param, **kwargs): # to convert the datetimes to timestamps with millisecond precision and # then zip the pairs back together. datetimes, values = zip(*timeseries) + if report_view == 'summary': + # Return sum count for pie-chart and table view + if aggregation == 'transaction_sum': + # When kind is change + return sum(values) * 0.000001 + elif aggregation == 'duration_minute': + return (sum(values) / 60) or 0 + else: + return sum(values) + timestamps = [ int(time.mktime(dt.timetuple()) * 1e3 + dt.microsecond / 1e3) for dt in datetimes @@ -150,6 +213,8 @@ def aggregate_timeseries(self, param, **kwargs): # Round the stats values when necessary. rounded_values = [] for value in values: + if param in TRANSFER_KINDS: + value = value * 0.000001 if round(value) != round(value, 2): rounded_values.append(round(value, 2)) else: @@ -369,3 +434,32 @@ def timeseries(self, key=None, **kwargs): if 'aggregation' not in kwargs: kwargs['aggregation'] = 'average_value' return self.aggregate_timeseries(key, **kwargs) + + +class TransferStatsClient(StatsClientBase): + """ Gather retailer transfer and recharge report """ + + def __init__(self, *args, **kwargs): + super(TransferStatsClient, self).__init__(*args, **kwargs) + + def timeseries(self, kind=None, **kwargs): + # Set queryset from subscriber role as retailer + kwargs['query'] = Q(subscriber__role='retailer') + return self.aggregate_timeseries(kind, **kwargs) + + +class TopUpStatsClient(StatsClientBase): + def __init__(self, *args, **kwargs): + super(TopUpStatsClient, self).__init__(*args, **kwargs) + + def timeseries(self, kind=None, **kwargs): + # Change is negative convert to compare + try: + raw_amount = [(float(denom) * -1 / 100000) for denom in + kwargs['extras'].split('-')] + kwargs['query'] = Q(change__gte=raw_amount[1]) & Q( + change__lte=raw_amount[0]) & Q(subscriber__role='retailer') + return self.aggregate_timeseries(kind, **kwargs) + except ValueError: + # If no denominations available in this network + raise ValueError('no denominations available in current network') diff --git a/cloud/endagaweb/stats_app/views.py b/cloud/endagaweb/stats_app/views.py index df6c88df..8b7b9ebd 100644 --- a/cloud/endagaweb/stats_app/views.py +++ b/cloud/endagaweb/stats_app/views.py @@ -26,13 +26,16 @@ CALL_KINDS = stats_client.CALL_KINDS + ['call'] GPRS_KINDS = ['total_data', 'uploaded_data', 'downloaded_data'] TIMESERIES_STAT_KEYS = stats_client.TIMESERIES_STAT_KEYS -VALID_STATS = SMS_KINDS + CALL_KINDS + GPRS_KINDS + TIMESERIES_STAT_KEYS +DENOMINATION_KINDS = stats_client.DENOMINATION_KINDS # Set valid intervals. INTERVALS = ['years', 'months', 'weeks', 'days', 'hours', 'minutes'] +TRANSFER_KINDS = stats_client.TRANSFER_KINDS +VALID_STATS = SMS_KINDS + CALL_KINDS + GPRS_KINDS + TIMESERIES_STAT_KEYS + \ + TRANSFER_KINDS + DENOMINATION_KINDS # Set valid aggregation types. AGGREGATIONS = ['count', 'duration', 'up_byte_count', 'down_byte_count', - 'average_value'] - + 'average_value', 'transaction_sum', 'transcation_count'] +REPORT_VIEWS = ['summary', 'list'] # Any requested start time earlier than this date will be set to this date. # This comes from a bug where UEs were generated at an astounding rate in @@ -52,6 +55,9 @@ def parse_query_params(params): 'stat-types': ['sms'], 'level-id': -1, 'aggregation': 'count', + 'report-view': 'list', + 'extras': [], + 'topup-percent': None, } # Override defaults with any query params that have been set, if the # query params are valid. @@ -74,10 +80,20 @@ def parse_query_params(params): # If nothing validated, we just leave the stat-types as the default. if validated_types: parsed_params['stat-types'] = validated_types + # Check if stat-type is dynamic currently for denominations + if params.has_key('dynamic-stat') and bool(params['dynamic-stat']): + parsed_params['stat-types'] = stat_types + # For filtering top topups as per percentage + if params.has_key('topup-percent'): + parsed_params['topup-percent'] = params['topup-percent'] if 'level-id' in params: parsed_params['level-id'] = int(params['level-id']) if 'aggregation' in params and params['aggregation'] in AGGREGATIONS: parsed_params['aggregation'] = params['aggregation'] + if 'extras' in params: + parsed_params['extras'] = params['extras'].split(',') + if 'report-view' in params and params['report-view'] in REPORT_VIEWS: + parsed_params['report-view'] = params['report-view'] return parsed_params @@ -125,7 +141,7 @@ def get(self, request, infrastructure_level): data = { 'results': [], } - for stat_type in params['stat-types']: + for index, stat_type in enumerate(params['stat-types']): # Setup the appropriate stats client, SMS, call or GPRS. if stat_type in SMS_KINDS: client_type = stats_client.SMSStatsClient @@ -135,6 +151,10 @@ def get(self, request, infrastructure_level): client_type = stats_client.GPRSStatsClient elif stat_type in TIMESERIES_STAT_KEYS: client_type = stats_client.TimeseriesStatsClient + elif stat_type in TRANSFER_KINDS: + client_type = stats_client.TransferStatsClient + else: + client_type = stats_client.TopUpStatsClient # Instantiate the client at an infrastructure level. if infrastructure_level == 'global': client = client_type('global') @@ -142,6 +162,10 @@ def get(self, request, infrastructure_level): client = client_type('network', params['level-id']) elif infrastructure_level == 'tower': client = client_type('tower', params['level-id']) + try: + extra_param = params['extras'][index] + except IndexError: + extra_param = None # Get timeseries results and append it to data. results = client.timeseries( stat_type, @@ -149,6 +173,9 @@ def get(self, request, infrastructure_level): start_time_epoch=params['start-time-epoch'], end_time_epoch=params['end-time-epoch'], aggregation=params['aggregation'], + report_view=params['report-view'], + extras=extra_param, + topup_percent=params['topup-percent'] ) data['results'].append({ "key": stat_type, diff --git a/cloud/endagaweb/templates/dashboard/layout.html b/cloud/endagaweb/templates/dashboard/layout.html index c4567a7f..d53eed11 100644 --- a/cloud/endagaweb/templates/dashboard/layout.html +++ b/cloud/endagaweb/templates/dashboard/layout.html @@ -74,6 +74,7 @@
  • Towers
  • Subscribers
  • Network
    • +
  • Reports