spark-cloudant/examples/python/CloudantApp.py at master · brianewilkins/spark-cloudant · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
#*******************************************************************************
# Copyright (c) 2015 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#******************************************************************************/
import pprint
from pyspark.sql import SQLContext
from pyspark import SparkContext, SparkConf

conf = SparkConf().setAppName("Cloudant Spark SQL External Datasource in Python")
# define coudant related configuration
conf.set("cloudant.host","ACCOUNT.cloudant.com")
conf.set("cloudant.username", "USERNAME")
conf.set("cloudant.password","PASSWORD")

sc = SparkContext(conf=conf)
sqlContext = SQLContext(sc)

print 'About to test com.cloudant.spark.CloudantRP for n_airportcodemapping'
sqlContext.sql("CREATE TEMPORARY TABLE airportTable USING com.cloudant.spark.CloudantRP OPTIONS ( database 'n_airportcodemapping')")

airportData = sqlContext.sql("SELECT _id, airportName FROM airportTable WHERE _id >= 'CAA' ORDER BY _id")
airportData.printSchema()
print 'Total # of rows in airportData: ' + str(airportData.count())
for code in airportData.collect():
	print code._id

print 'About to test com.cloudant.spark.CloudantRP for n_booking'
sqlContext.sql(" CREATE TEMPORARY TABLE bookingTable USING com.cloudant.spark.CloudantRP OPTIONS ( database 'n_booking')")

#bookingData = sqlContext.sql("SELECT customerId, dateOfBooking FROM bookingTable WHERE customerId = 'uid0@email.com'") -- ArrayIndexOutOfBoundsException on 1.4.1
#bookingData.printSchema()
#for code in bookingData.collect():
#	print code.customerId
#	print code.dateOfBooking


print 'About to test com.cloudant.spark.CloudantPrunedFilteredRP for n_airportcodemapping'
sqlContext.sql(" CREATE TEMPORARY TABLE airportTable1 USING com.cloudant.spark.CloudantPrunedFilteredRP OPTIONS ( database 'n_airportcodemapping')")

airportData = sqlContext.sql("SELECT _id, airportName FROM airportTable1 WHERE _id >= 'CAA' AND _id <= 'GAA' ORDER BY _id")
airportData.printSchema()
print 'Total # of rows in airportData: ' + str(airportData.count())
for code in airportData.collect():
	print code._id

print 'About to test com.cloudant.spark.CloudantPrunedFilteredRP for n_booking'
sqlContext.sql(" CREATE TEMPORARY TABLE bookingTable1 USING com.cloudant.spark.CloudantPrunedFilteredRP OPTIONS ( database 'n_booking')")

bookingData = sqlContext.sql("SELECT customerId, dateOfBooking FROM bookingTable1 WHERE customerId = 'uid0@email.com'")
bookingData.printSchema()
for code in bookingData.collect():
	print code.customerId
	print code.dateOfBooking

print 'About to test com.cloudant.spark.CloudantPrunedFilteredRP for flight with index'
sqlContext.sql(" CREATE TEMPORARY TABLE flightTable1 USING com.cloudant.spark.CloudantPrunedFilteredRP OPTIONS ( database 'n_flight', index '_design/view/_search/n_flights')")

flightData = sqlContext.sql("SELECT flightSegmentId, scheduledDepartureTime FROM flightTable1 WHERE flightSegmentId >'AA9' AND flightSegmentId<'AA95'")
flightData.printSchema()
for code in flightData.collect():
	print 'Flight {0} on {1}'.format(code.flightSegmentId, code.scheduledDepartureTime)


print 'About to test com.cloudant.spark.CloudantPartitionedPrunedFilteredRP for n_airportcodemapping'
sqlContext.sql(" CREATE TEMPORARY TABLE airportTable2 USING com.cloudant.spark.CloudantPartitionedPrunedFilteredRP OPTIONS ( database 'n_airportcodemapping')")

airportData = sqlContext.sql("SELECT _id, airportName FROM airportTable2 WHERE _id >= 'CAA' AND _id <= 'GAA' ORDER BY _id")
airportData.printSchema()
for code in airportData.collect():
	print code._id

print 'About to test com.cloudant.spark.CloudantPartitionedPrunedFilteredRP for n_booking'
sqlContext.sql(" CREATE TEMPORARY TABLE bookingTable2 USING com.cloudant.spark.CloudantPartitionedPrunedFilteredRP OPTIONS ( database 'n_booking')")

bookingData = sqlContext.sql("SELECT customerId, dateOfBooking FROM bookingTable2 WHERE customerId = 'uid0@email.com'")
bookingData.printSchema()
for code in bookingData.collect():
	print 'Booking for {0} on {1}'.format(code.customerId,code.dateOfBooking)

print 'About to test com.cloudant.spark.CloudantPartitionedPrunedFilteredRP for flight with index'
sqlContext.sql(" CREATE TEMPORARY TABLE flightTable2 USING com.cloudant.spark.CloudantPartitionedPrunedFilteredRP OPTIONS ( database 'n_flight', index '_design/view/_search/n_flights')")

flightData = sqlContext.sql("SELECT flightSegmentId, scheduledDepartureTime FROM flightTable2 WHERE flightSegmentId >'AA9' AND flightSegmentId<'AA95'")
flightData.printSchema()
for code in flightData.collect():
	print 'Flight {0} on {1}'.format(code.flightSegmentId, code.scheduledDepartureTime)