From fde669ed6a2c7f210d7856ff6b5b7adc0524d23f Mon Sep 17 00:00:00 2001 From: TonyTang Date: Mon, 24 Jun 2019 15:48:20 -0700 Subject: [PATCH] Support url fitlers and header filters (#1) --- build.gradle | 17 ++-- gradle/wrapper/gradle-wrapper.properties | 2 +- library-no-op/build.gradle | 2 +- .../chuck/ChuckInterceptor.java | 13 +++ .../internal/data/ChuckContentProvider.java | 26 ++++++ library/build.gradle | 8 +- library/proguard-rules.pro | 3 +- .../chuck/ChuckInterceptor.java | 93 +++++++++++++++++-- .../internal/data/ChuckContentProvider.java | 28 ++++++ .../chuck/internal/data/HttpTransaction.java | 27 ++++-- .../chuck/internal/support/FormatUtils.java | 11 ++- sample/build.gradle | 16 ++-- sample/src/main/AndroidManifest.xml | 11 ++- .../chuck/sample/MainActivity.java | 44 ++++++++- sample/src/main/res/layout/activity_main.xml | 9 +- sample/src/main/res/values/strings.xml | 1 + settings.gradle | 3 +- 17 files changed, 268 insertions(+), 46 deletions(-) create mode 100644 library-no-op/src/main/java/com/readystatesoftware/chuck/internal/data/ChuckContentProvider.java diff --git a/build.gradle b/build.gradle index 7a82cc95..b591f2dc 100644 --- a/build.gradle +++ b/build.gradle @@ -3,9 +3,12 @@ buildscript { repositories { jcenter() + google() + maven { url 'https://jitpack.io' } + } dependencies { - classpath 'com.android.tools.build:gradle:2.3.3' + classpath 'com.android.tools.build:gradle:3.2.1' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files @@ -21,6 +24,8 @@ allprojects { maven { url "https://maven.google.com" } + maven { url 'https://jitpack.io' } + google() } } @@ -30,10 +35,10 @@ task clean(type: Delete) { ext { minSdkVersion = 16 - targetSdkVersion = 26 - compileSdkVersion = 26 - buildToolsVersion = '26.0.1' + targetSdkVersion = 28 + compileSdkVersion = 28 + buildToolsVersion = '28.0.3' - supportLibVersion = '25.3.1' - okhttp3Version = '3.6.0' + supportLibVersion = '28.0.0' + okhttp3Version = '3.11.0' } \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 5192af6d..f65824e5 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-all.zip diff --git a/library-no-op/build.gradle b/library-no-op/build.gradle index 8dea4692..46750ceb 100644 --- a/library-no-op/build.gradle +++ b/library-no-op/build.gradle @@ -10,7 +10,7 @@ android { } dependencies { - compile "com.squareup.okhttp3:okhttp:$okhttp3Version" + implementation "com.squareup.okhttp3:okhttp:$okhttp3Version" } apply from: rootProject.file('gradle/gradle-mvn-push.gradle') \ No newline at end of file diff --git a/library-no-op/src/main/java/com/readystatesoftware/chuck/ChuckInterceptor.java b/library-no-op/src/main/java/com/readystatesoftware/chuck/ChuckInterceptor.java index 0f5585f1..3e37d892 100644 --- a/library-no-op/src/main/java/com/readystatesoftware/chuck/ChuckInterceptor.java +++ b/library-no-op/src/main/java/com/readystatesoftware/chuck/ChuckInterceptor.java @@ -18,6 +18,7 @@ import android.content.Context; import java.io.IOException; +import java.util.List; import okhttp3.Interceptor; import okhttp3.Request; @@ -28,6 +29,18 @@ */ public final class ChuckInterceptor implements Interceptor { + public ChuckInterceptor setFilterBody(boolean filterBoby) { + return this; + } + + public ChuckInterceptor setFilterHeaderList(List keyWordHeaderList) { + return this; + } + + public ChuckInterceptor setFilterUrlList(List keyWordUrlList) { + return this; + } + public enum Period { ONE_HOUR, ONE_DAY, diff --git a/library-no-op/src/main/java/com/readystatesoftware/chuck/internal/data/ChuckContentProvider.java b/library-no-op/src/main/java/com/readystatesoftware/chuck/internal/data/ChuckContentProvider.java new file mode 100644 index 00000000..10302ada --- /dev/null +++ b/library-no-op/src/main/java/com/readystatesoftware/chuck/internal/data/ChuckContentProvider.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2017 Jeff Gilfelt. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.readystatesoftware.chuck.internal.data; + +import android.content.Context; + + +public class ChuckContentProvider { + + public static String export(Context context) { + return null; + } +} diff --git a/library/build.gradle b/library/build.gradle index ed58a517..0f535434 100644 --- a/library/build.gradle +++ b/library/build.gradle @@ -11,10 +11,10 @@ android { } dependencies { - compile 'com.google.code.gson:gson:2.8.0' - compile "com.squareup.okhttp3:okhttp:$okhttp3Version" - compile 'nl.qbusict:cupboard:2.2.0' - compile "com.android.support:design:$supportLibVersion" + implementation 'com.google.code.gson:gson:2.8.4' + implementation "com.squareup.okhttp3:okhttp:$okhttp3Version" + implementation 'nl.qbusict:cupboard:2.2.0' + implementation "com.android.support:design:$supportLibVersion" } apply from: rootProject.file('gradle/gradle-mvn-push.gradle') diff --git a/library/proguard-rules.pro b/library/proguard-rules.pro index 767e267c..8026f159 100644 --- a/library/proguard-rules.pro +++ b/library/proguard-rules.pro @@ -1,2 +1,3 @@ -keep class com.readystatesoftware.chuck.internal.data.HttpTransaction { *; } --keep class android.support.v7.widget.SearchView { *; } \ No newline at end of file +-keep class android.support.v7.widget.SearchView { *; } +-dontnote com.readystatesoftware.chuck.internal.data.HttpTransaction diff --git a/library/src/main/java/com/readystatesoftware/chuck/ChuckInterceptor.java b/library/src/main/java/com/readystatesoftware/chuck/ChuckInterceptor.java index 433d9207..0206afb3 100644 --- a/library/src/main/java/com/readystatesoftware/chuck/ChuckInterceptor.java +++ b/library/src/main/java/com/readystatesoftware/chuck/ChuckInterceptor.java @@ -31,7 +31,10 @@ import java.nio.charset.Charset; import java.nio.charset.UnsupportedCharsetException; import java.util.Date; +import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import okhttp3.Headers; import okhttp3.Interceptor; @@ -78,6 +81,10 @@ public enum Period { private final NotificationHelper notificationHelper; private RetentionManager retentionManager; private boolean showNotification; + private List filterUrlList; + private List filterHeaderList; + private boolean filterBody; + private long maxContentLength = 250000L; /** @@ -101,6 +108,36 @@ public ChuckInterceptor showNotification(boolean show) { return this; } + /** + * set all the http response body should be filtered. + * @param filterBody true if you want to filter all http response body. + */ + public ChuckInterceptor setFilterBody(boolean filterBody) { + this.filterBody = filterBody; + return this; + } + + /** + * set the url key word list to stop chuck from caching its data. + * @param keyWordUrlList the url level key word list that is to stop chuck from caching its data. All of these data including + * request headers, request params, request url params, response header, response body will not be cached. + * @return The {@link ChuckInterceptor} instance. + */ + public ChuckInterceptor setFilterUrlList(List keyWordUrlList) { + this.filterUrlList = keyWordUrlList; + return this; + } + + /** + * set the header key word list to stop chuck from caching its data. + * @param keyWordHeaderList the header level key word list that is to stop chuck from caching the value of such header. + * @return The {@link ChuckInterceptor} instance. + */ + public ChuckInterceptor setFilterHeaderList(List keyWordHeaderList) { + this.filterHeaderList = keyWordHeaderList; + return this; + } + /** * Set the maximum length for request and response content before it is truncated. * Warning: setting this value too high may cause unexpected results. @@ -112,7 +149,7 @@ public ChuckInterceptor maxContentLength(long max) { this.maxContentLength = max; return this; } - + /** * Set the retention period for HTTP transaction data captured by this interceptor. * The default is one week. @@ -125,6 +162,25 @@ public ChuckInterceptor retainDataFor(Period period) { return this; } + private String replacedUrl(String url) { + if (this.filterUrlList ==null){ + return null; + } + for (String keyWord : filterUrlList) { + if (matches(url, keyWord)){ + int endIndex = url.lastIndexOf("?"); + return endIndex > 0 ? url.substring(0, endIndex) : url; + } + } + return null; + } + + private boolean matches(String url, String regex) { + Pattern p = Pattern.compile(regex); + Matcher m = p.matcher(url); + return m.find(); + } + @Override public Response intercept(Chain chain) throws IOException { Request request = chain.request(); @@ -135,9 +191,18 @@ public ChuckInterceptor retainDataFor(Period period) { transaction.setRequestDate(new Date()); transaction.setMethod(request.method()); - transaction.setUrl(request.url().toString()); - transaction.setRequestHeaders(request.headers()); + String url = request.url().toString(); + String replacedUrl = replacedUrl(url); + boolean urlNeedToBeFiltered = replacedUrl!=null; + if (!urlNeedToBeFiltered){ + transaction.setUrl(url); + }else { + transaction.setUrl(replacedUrl); + } + if (!urlNeedToBeFiltered){ + transaction.setRequestHeaders(request.headers(), filterHeaderList); + } if (hasRequestBody) { if (requestBody.contentType() != null) { transaction.setRequestContentType(requestBody.contentType().toString()); @@ -158,7 +223,9 @@ public ChuckInterceptor retainDataFor(Period period) { charset = contentType.charset(UTF8); } if (isPlaintext(buffer)) { - transaction.setRequestBody(readFromBuffer(buffer, charset)); + if (!urlNeedToBeFiltered){ + transaction.setRequestBody(readFromBuffer(buffer, charset)); + } } else { transaction.setResponseBodyIsPlainText(false); } @@ -179,19 +246,21 @@ public ChuckInterceptor retainDataFor(Period period) { ResponseBody responseBody = response.body(); - transaction.setRequestHeaders(response.request().headers()); // includes headers added later in the chain + if (!urlNeedToBeFiltered) { + transaction.setRequestHeaders(response.request().headers(), filterHeaderList); // includes headers added later in the chain + } transaction.setResponseDate(new Date()); transaction.setTookMs(tookMs); transaction.setProtocol(response.protocol().toString()); transaction.setResponseCode(response.code()); transaction.setResponseMessage(response.message()); - transaction.setResponseContentLength(responseBody.contentLength()); if (responseBody.contentType() != null) { transaction.setResponseContentType(responseBody.contentType().toString()); } - transaction.setResponseHeaders(response.headers()); - + if (!urlNeedToBeFiltered) { + transaction.setResponseHeaders(response.headers(), filterHeaderList); + } transaction.setResponseBodyIsPlainText(!bodyHasUnsupportedEncoding(response.headers())); if (HttpHeaders.hasBody(response) && transaction.responseBodyIsPlainText()) { BufferedSource source = getNativeSource(response); @@ -208,7 +277,13 @@ public ChuckInterceptor retainDataFor(Period period) { } } if (isPlaintext(buffer)) { - transaction.setResponseBody(readFromBuffer(buffer.clone(), charset)); + if (!urlNeedToBeFiltered) { + if (!filterBody){ + transaction.setResponseBody(readFromBuffer(buffer.clone(), charset)); + }else { + transaction.setResponseBody("****** body has been hidden ******"); + } + } } else { transaction.setResponseBodyIsPlainText(false); } diff --git a/library/src/main/java/com/readystatesoftware/chuck/internal/data/ChuckContentProvider.java b/library/src/main/java/com/readystatesoftware/chuck/internal/data/ChuckContentProvider.java index 969a899b..5bbe6226 100644 --- a/library/src/main/java/com/readystatesoftware/chuck/internal/data/ChuckContentProvider.java +++ b/library/src/main/java/com/readystatesoftware/chuck/internal/data/ChuckContentProvider.java @@ -26,6 +26,14 @@ import android.net.Uri; import android.support.annotation.NonNull; import android.support.annotation.Nullable; +import android.support.annotation.WorkerThread; + +import com.readystatesoftware.chuck.internal.support.JsonConvertor; + +import java.util.ArrayList; +import java.util.List; + +import nl.qbusict.cupboard.QueryResultIterable; public class ChuckContentProvider extends ContentProvider { @@ -37,6 +45,26 @@ public class ChuckContentProvider extends ContentProvider { private ChuckDbOpenHelper databaseHelper; + + @WorkerThread + public static String export(Context context){ + ChuckDbOpenHelper chuckDbOpenHelper = new ChuckDbOpenHelper(context); + SQLiteDatabase readableDatabase = chuckDbOpenHelper.getReadableDatabase(); + QueryResultIterable iterable = LocalCupboard.getInstance().withDatabase(readableDatabase).query(HttpTransaction.class).query(); + List list = getListFromQueryResultIterator(iterable); + return JsonConvertor.getInstance().toJson(list); + } + + private static List getListFromQueryResultIterator(QueryResultIterable iter) { + + final List transactionArrayList = new ArrayList<>(); + for (HttpTransaction bunny : iter) { + transactionArrayList.add(bunny); + } + iter.close(); + return transactionArrayList; + } + @Override public void attachInfo(Context context, ProviderInfo info) { super.attachInfo(context, info); diff --git a/library/src/main/java/com/readystatesoftware/chuck/internal/data/HttpTransaction.java b/library/src/main/java/com/readystatesoftware/chuck/internal/data/HttpTransaction.java index a39a843f..a1e981f3 100644 --- a/library/src/main/java/com/readystatesoftware/chuck/internal/data/HttpTransaction.java +++ b/library/src/main/java/com/readystatesoftware/chuck/internal/data/HttpTransaction.java @@ -250,8 +250,8 @@ public String getScheme() { return scheme; } - public void setRequestHeaders(Headers headers) { - setRequestHeaders(toHttpHeaderList(headers)); + public void setRequestHeaders(Headers headers, List filterHeaderList) { + setRequestHeaders(toHttpHeaderList(headers, filterHeaderList)); } public void setRequestHeaders(List headers) { @@ -267,8 +267,8 @@ public String getRequestHeadersString(boolean withMarkup) { return FormatUtils.formatHeaders(getRequestHeaders(), withMarkup); } - public void setResponseHeaders(Headers headers) { - setResponseHeaders(toHttpHeaderList(headers)); + public void setResponseHeaders(Headers headers, List filterHeaderList) { + setResponseHeaders(toHttpHeaderList(headers, filterHeaderList)); } public void setResponseHeaders(List headers) { @@ -349,14 +349,29 @@ public boolean isSsl() { return scheme.toLowerCase().equals("https"); } - private List toHttpHeaderList(Headers headers) { + private List toHttpHeaderList(Headers headers, List filterHeaderList) { List httpHeaders = new ArrayList<>(); for (int i = 0, count = headers.size(); i < count; i++) { - httpHeaders.add(new HttpHeader(headers.name(i), headers.value(i))); + httpHeaders.add(new HttpHeader(headers.name(i), headerFiltered(filterHeaderList, headers.name(i)) ? "*****" : headers.value(i))); } return httpHeaders; } + private boolean headerFiltered(List filterHeaderList, String name) { + return filterHeaderList!=null && containsCaseInsensitive(filterHeaderList, name); + } + + public boolean containsCaseInsensitive(List stringList, String name){ + for (String string : stringList){ + if (string.equalsIgnoreCase(name)){ + return true; + } + } + return false; + } + + + private String formatBody(String body, String contentType) { if (contentType != null && contentType.toLowerCase().contains("json")) { return FormatUtils.formatJson(body); diff --git a/library/src/main/java/com/readystatesoftware/chuck/internal/support/FormatUtils.java b/library/src/main/java/com/readystatesoftware/chuck/internal/support/FormatUtils.java index 3e58f301..35c70134 100644 --- a/library/src/main/java/com/readystatesoftware/chuck/internal/support/FormatUtils.java +++ b/library/src/main/java/com/readystatesoftware/chuck/internal/support/FormatUtils.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Locale; +import javax.xml.XMLConstants; import javax.xml.transform.OutputKeys; import javax.xml.transform.Source; import javax.xml.transform.Transformer; @@ -55,7 +56,7 @@ public static String formatByteCount(long bytes, boolean si) { int unit = si ? 1000 : 1024; if (bytes < unit) return bytes + " B"; int exp = (int) (Math.log(bytes) / Math.log(unit)); - String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp-1) + (si ? "" : "i"); + String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i"); return String.format(Locale.US, "%.1f %sB", bytes / Math.pow(unit, exp), pre); } @@ -71,13 +72,17 @@ public static String formatJson(String json) { public static String formatXml(String xml) { try { - Transformer serializer = SAXTransformerFactory.newInstance().newTransformer(); + SAXTransformerFactory transformerFactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance(); + transformerFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + transformerFactory.setAttribute("http://javax.xml.XMLConstants/property/accessExternalDTD", ""); + transformerFactory.setAttribute("http://javax.xml.XMLConstants/property/accessExternalStylesheet", ""); + Transformer serializer = transformerFactory.newTransformer(); serializer.setOutputProperty(OutputKeys.INDENT, "yes"); serializer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); Source xmlSource = new SAXSource(new InputSource(new ByteArrayInputStream(xml.getBytes()))); StreamResult res = new StreamResult(new ByteArrayOutputStream()); serializer.transform(xmlSource, res); - return new String(((ByteArrayOutputStream)res.getOutputStream()).toByteArray()); + return new String(((ByteArrayOutputStream) res.getOutputStream()).toByteArray()); } catch (Exception e) { return xml; } diff --git a/sample/build.gradle b/sample/build.gradle index 018e4fd6..432a063c 100644 --- a/sample/build.gradle +++ b/sample/build.gradle @@ -25,11 +25,13 @@ android { } dependencies { - debugCompile project(':library') - releaseCompile project(':library-no-op') - compile "com.android.support:design:26.0.0" - compile "com.android.support:appcompat-v7:26.0.0" - compile "com.squareup.okhttp3:logging-interceptor:$okhttp3Version" - compile "com.squareup.retrofit2:retrofit:2.2.0" - compile "com.squareup.retrofit2:converter-gson:2.2.0" + debugImplementation project(':library') + releaseImplementation project(':library-no-op') + implementation "com.android.support:design:28.0.0" + implementation "com.android.support:appcompat-v7:28.0.0" + implementation "com.squareup.okhttp3:logging-interceptor:$okhttp3Version" + implementation "com.squareup.retrofit2:retrofit:2.5.0" + implementation "com.squareup.retrofit2:converter-gson:2.5.0" + implementation("com.google.code.gson:gson:2.8.4") + } diff --git a/sample/src/main/AndroidManifest.xml b/sample/src/main/AndroidManifest.xml index 89fd8c28..7bd77d0c 100644 --- a/sample/src/main/AndroidManifest.xml +++ b/sample/src/main/AndroidManifest.xml @@ -15,20 +15,21 @@ ~ limitations under the License. --> + package="com.readystatesoftware.chuck.sample"> - + + android:theme="@style/AppTheme" + android:usesCleartextTraffic="true"> - - + + diff --git a/sample/src/main/java/com/readystatesoftware/chuck/sample/MainActivity.java b/sample/src/main/java/com/readystatesoftware/chuck/sample/MainActivity.java index d56c231f..d2dadd38 100644 --- a/sample/src/main/java/com/readystatesoftware/chuck/sample/MainActivity.java +++ b/sample/src/main/java/com/readystatesoftware/chuck/sample/MainActivity.java @@ -19,10 +19,15 @@ import android.os.Bundle; import android.support.annotation.Nullable; import android.support.v7.app.AppCompatActivity; +import android.util.Log; import android.view.View; import com.readystatesoftware.chuck.Chuck; import com.readystatesoftware.chuck.ChuckInterceptor; +import com.readystatesoftware.chuck.internal.data.ChuckContentProvider; + +import java.util.ArrayList; +import java.util.List; import okhttp3.OkHttpClient; import okhttp3.logging.HttpLoggingInterceptor; @@ -48,16 +53,53 @@ public void onClick(View view) { launchChuckDirectly(); } }); + findViewById(R.id.export_json_data).setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View view) { + exportJsonData(); + } + }); + } + + private void exportJsonData() { + new Thread(new Runnable() { + @Override + public void run() { + String export = ChuckContentProvider.export(MainActivity.this); + Log.v("MainActivity", "Exported json:" + export); + } + }).run(); } private OkHttpClient getClient(Context context) { + + ChuckInterceptor interceptor = new ChuckInterceptor(context) + .retainDataFor(ChuckInterceptor.Period.ONE_DAY) + .showNotification(true); + addFilters(interceptor); + return new OkHttpClient.Builder() // Add a ChuckInterceptor instance to your OkHttp client - .addInterceptor(new ChuckInterceptor(context)) + .addInterceptor(interceptor) .addInterceptor(new HttpLoggingInterceptor().setLevel(HttpLoggingInterceptor.Level.BODY)) .build(); } + private void addFilters(ChuckInterceptor interceptor) { + List keyWordHeaderList = new ArrayList<>(); + keyWordHeaderList.add("Access-Control-Allow-Credentials"); + keyWordHeaderList.add("Access-Control-Allow-Origin"); + + List keyWordUrlList = new ArrayList<>(); + keyWordUrlList.add("cookies"); + keyWordUrlList.add("auth"); + + + interceptor.setFilterBody(true) + .setFilterHeaderList(keyWordHeaderList) + .setFilterUrlList(keyWordUrlList); + } + private void launchChuckDirectly() { // Optionally launch Chuck directly from your own app UI startActivity(Chuck.getLaunchIntent(this)); diff --git a/sample/src/main/res/layout/activity_main.xml b/sample/src/main/res/layout/activity_main.xml index 702af0c6..56865973 100644 --- a/sample/src/main/res/layout/activity_main.xml +++ b/sample/src/main/res/layout/activity_main.xml @@ -38,7 +38,14 @@ android:layout_width="wrap_content" android:layout_height="wrap_content" android:layout_below="@+id/do_http" + android:id="@+id/launch_chuck_directly"/> + +