1265 lines
42 KiB
JavaScript
1265 lines
42 KiB
JavaScript
/*******************************************************************************
|
|
|
|
uBlock Origin - a comprehensive, efficient content blocker
|
|
Copyright (C) 2014-present Raymond Hill
|
|
|
|
This program is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program. If not, see {http://www.gnu.org/licenses/}.
|
|
|
|
Home: https://github.com/gorhill/uBlock
|
|
*/
|
|
|
|
/******************************************************************************/
|
|
|
|
import * as fc from './filtering-context.js';
|
|
import * as sfp from './static-filtering-parser.js';
|
|
import {
|
|
sessionFirewall,
|
|
sessionSwitches,
|
|
sessionURLFiltering,
|
|
} from './filtering-engines.js';
|
|
import htmlFilteringEngine from './html-filtering.js';
|
|
import httpheaderFilteringEngine from './httpheader-filtering.js';
|
|
import { isNetworkURI } from './uri-utils.js';
|
|
import logger from './logger.js';
|
|
import scriptletFilteringEngine from './scriptlet-filtering.js';
|
|
import staticNetFilteringEngine from './static-net-filtering.js';
|
|
import textEncode from './text-encode.js';
|
|
import µb from './background.js';
|
|
|
|
/******************************************************************************/
|
|
|
|
// Platform-specific behavior.
|
|
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/42
|
|
// https://bugzilla.mozilla.org/show_bug.cgi?id=1376932
|
|
// Add proper version number detection once issue is fixed in Firefox.
|
|
let dontCacheResponseHeaders =
|
|
vAPI.webextFlavor.soup.has('firefox');
|
|
|
|
// The real actual webextFlavor value may not be set in stone, so listen
|
|
// for possible future changes.
|
|
window.addEventListener('webextFlavor', function() {
|
|
dontCacheResponseHeaders =
|
|
vAPI.webextFlavor.soup.has('firefox');
|
|
}, { once: true });
|
|
|
|
/******************************************************************************/
|
|
|
|
const patchLocalRedirectURL = url => url.charCodeAt(0) === 0x2F /* '/' */
|
|
? vAPI.getURL(url)
|
|
: url;
|
|
|
|
/******************************************************************************/
|
|
|
|
// Intercept and filter web requests.
|
|
|
|
const onBeforeRequest = function(details) {
|
|
const fctxt = µb.filteringContext.fromWebrequestDetails(details);
|
|
|
|
// Special handling for root document.
|
|
// https://github.com/chrisaljoudi/uBlock/issues/1001
|
|
// This must be executed regardless of whether the request is
|
|
// behind-the-scene
|
|
if ( fctxt.itype === fctxt.MAIN_FRAME ) {
|
|
return onBeforeRootFrameRequest(fctxt);
|
|
}
|
|
|
|
// Special treatment: behind-the-scene requests
|
|
const tabId = details.tabId;
|
|
if ( tabId < 0 ) {
|
|
return onBeforeBehindTheSceneRequest(fctxt);
|
|
}
|
|
|
|
// Lookup the page store associated with this tab id.
|
|
let pageStore = µb.pageStoreFromTabId(tabId);
|
|
if ( pageStore === null ) {
|
|
const tabContext = µb.tabContextManager.mustLookup(tabId);
|
|
if ( tabContext.tabId < 0 ) {
|
|
return onBeforeBehindTheSceneRequest(fctxt);
|
|
}
|
|
vAPI.tabs.onNavigation({ tabId, frameId: 0, url: tabContext.rawURL });
|
|
pageStore = µb.pageStoreFromTabId(tabId);
|
|
}
|
|
|
|
const result = pageStore.filterRequest(fctxt);
|
|
|
|
pageStore.journalAddRequest(fctxt, result);
|
|
|
|
if ( logger.enabled ) {
|
|
fctxt.setRealm('network').toLogger();
|
|
}
|
|
|
|
// Redirected
|
|
|
|
if ( fctxt.redirectURL !== undefined ) {
|
|
return { redirectUrl: patchLocalRedirectURL(fctxt.redirectURL) };
|
|
}
|
|
|
|
// Not redirected
|
|
|
|
// Blocked
|
|
if ( result === 1 ) {
|
|
return { cancel: true };
|
|
}
|
|
|
|
// Not blocked
|
|
if (
|
|
fctxt.itype === fctxt.SUB_FRAME &&
|
|
details.parentFrameId !== -1 &&
|
|
details.aliasURL === undefined
|
|
) {
|
|
pageStore.setFrameURL(details);
|
|
}
|
|
|
|
if ( result === 2 ) {
|
|
return { cancel: false };
|
|
}
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
const onBeforeRootFrameRequest = function(fctxt) {
|
|
const requestURL = fctxt.url;
|
|
|
|
// Special handling for root document.
|
|
// https://github.com/chrisaljoudi/uBlock/issues/1001
|
|
// This must be executed regardless of whether the request is
|
|
// behind-the-scene
|
|
const requestHostname = fctxt.getHostname();
|
|
let result = 0;
|
|
let logData;
|
|
|
|
// If the site is whitelisted, disregard strict blocking
|
|
const trusted = µb.getNetFilteringSwitch(requestURL) === false;
|
|
if ( trusted ) {
|
|
result = 2;
|
|
if ( logger.enabled ) {
|
|
logData = { engine: 'u', result: 2, raw: 'whitelisted' };
|
|
}
|
|
}
|
|
|
|
// Permanently unrestricted?
|
|
if (
|
|
result === 0 &&
|
|
sessionSwitches.evaluateZ('no-strict-blocking', requestHostname)
|
|
) {
|
|
result = 2;
|
|
if ( logger.enabled ) {
|
|
logData = {
|
|
engine: 'u',
|
|
result: 2,
|
|
raw: `no-strict-blocking: ${sessionSwitches.z} true`
|
|
};
|
|
}
|
|
}
|
|
|
|
// Temporarily whitelisted?
|
|
if ( result === 0 && strictBlockBypasser.isBypassed(requestHostname) ) {
|
|
result = 2;
|
|
if ( logger.enabled ) {
|
|
logData = {
|
|
engine: 'u',
|
|
result: 2,
|
|
raw: 'no-strict-blocking: true (temporary)'
|
|
};
|
|
}
|
|
}
|
|
|
|
// Static filtering
|
|
if ( result === 0 ) {
|
|
({ result, logData } = shouldStrictBlock(fctxt, logger.enabled));
|
|
}
|
|
|
|
const pageStore = µb.bindTabToPageStore(fctxt.tabId, 'beforeRequest');
|
|
if ( pageStore !== null ) {
|
|
pageStore.journalAddRootFrame('uncommitted', requestURL);
|
|
pageStore.journalAddRequest(fctxt, result);
|
|
}
|
|
|
|
if ( logger.enabled ) {
|
|
fctxt.setFilter(logData);
|
|
}
|
|
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/760
|
|
// Redirect non-blocked request?
|
|
if ( result !== 1 && trusted === false && pageStore !== null ) {
|
|
pageStore.redirectNonBlockedRequest(fctxt);
|
|
}
|
|
|
|
if ( logger.enabled ) {
|
|
fctxt.setRealm('network').toLogger();
|
|
}
|
|
|
|
// Redirected
|
|
|
|
if ( fctxt.redirectURL !== undefined ) {
|
|
return { redirectUrl: patchLocalRedirectURL(fctxt.redirectURL) };
|
|
}
|
|
|
|
// Not blocked
|
|
|
|
if ( result !== 1 ) { return; }
|
|
|
|
// No log data means no strict blocking (because we need to report why
|
|
// the blocking occurs.
|
|
if ( logData === undefined ) { return; }
|
|
|
|
// Blocked
|
|
|
|
const query = encodeURIComponent(JSON.stringify({
|
|
url: requestURL,
|
|
hn: requestHostname,
|
|
dn: fctxt.getDomain() || requestHostname,
|
|
fs: logData.raw
|
|
}));
|
|
|
|
vAPI.tabs.replace(
|
|
fctxt.tabId,
|
|
vAPI.getURL('document-blocked.html?details=') + query
|
|
);
|
|
|
|
return { cancel: true };
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
// Strict blocking through static filtering
|
|
//
|
|
// https://github.com/chrisaljoudi/uBlock/issues/1128
|
|
// Do not block if the match begins after the hostname,
|
|
// except when the filter is specifically of type `other`.
|
|
// https://github.com/gorhill/uBlock/issues/490
|
|
// Removing this for the time being, will need a new, dedicated type.
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/1501
|
|
// Support explicit exception filters.
|
|
//
|
|
// Let result of match for specific `document` type be `rs`
|
|
// Let result of match for no specific type be `rg` *after* going through
|
|
// confirmation necessary for implicit matches
|
|
// Let `important` be `i`
|
|
// Let final result be logical combination of `rs` and `rg` as follow:
|
|
//
|
|
// | rs |
|
|
// +--------+--------+--------+--------|
|
|
// | 0 | 1 | 1i | 2 |
|
|
// --------+--------+--------+--------+--------+--------|
|
|
// | 0 | rg | rs | rs | rs |
|
|
// rg | 1 | rg | rs | rs | rs |
|
|
// | 1i | rg | rg | rs | rg |
|
|
// | 2 | rg | rg | rs | rs |
|
|
// --------+--------+--------+--------+--------+--------+
|
|
|
|
const shouldStrictBlock = function(fctxt, loggerEnabled) {
|
|
const snfe = staticNetFilteringEngine;
|
|
|
|
// Explicit filtering: `document` option
|
|
const rs = snfe.matchRequest(fctxt, 0b0011);
|
|
const is = rs === 1 && snfe.isBlockImportant();
|
|
let lds;
|
|
if ( rs !== 0 || loggerEnabled ) {
|
|
lds = snfe.toLogData();
|
|
}
|
|
|
|
// | rs |
|
|
// +--------+--------+--------+--------|
|
|
// | 0 | 1 | 1i | 2 |
|
|
// --------+--------+--------+--------+--------+--------|
|
|
// | 0 | rg | rs | x | rs |
|
|
// rg | 1 | rg | rs | x | rs |
|
|
// | 1i | rg | rg | x | rg |
|
|
// | 2 | rg | rg | x | rs |
|
|
// --------+--------+--------+--------+--------+--------+
|
|
if ( rs === 1 && is ) {
|
|
return { result: rs, logData: lds };
|
|
}
|
|
|
|
// Implicit filtering: no `document` option
|
|
fctxt.type = 'no_type';
|
|
let rg = snfe.matchRequest(fctxt, 0b0011);
|
|
fctxt.type = 'main_frame';
|
|
const ig = rg === 1 && snfe.isBlockImportant();
|
|
let ldg;
|
|
if ( rg !== 0 || loggerEnabled ) {
|
|
ldg = snfe.toLogData();
|
|
if ( rg === 1 && validateStrictBlock(fctxt, ldg) === false ) {
|
|
rg = 0; ldg = undefined;
|
|
}
|
|
}
|
|
|
|
// | rs |
|
|
// +--------+--------+--------+--------|
|
|
// | 0 | 1 | 1i | 2 |
|
|
// --------+--------+--------+--------+--------+--------|
|
|
// | 0 | x | rs | - | rs |
|
|
// rg | 1 | x | rs | - | rs |
|
|
// | 1i | x | x | - | x |
|
|
// | 2 | x | x | - | rs |
|
|
// --------+--------+--------+--------+--------+--------+
|
|
if ( rs === 0 || rg === 1 && ig || rg === 2 && rs !== 2 ) {
|
|
return { result: rg, logData: ldg };
|
|
}
|
|
|
|
// | rs |
|
|
// +--------+--------+--------+--------|
|
|
// | 0 | 1 | 1i | 2 |
|
|
// --------+--------+--------+--------+--------+--------|
|
|
// | 0 | - | x | - | x |
|
|
// rg | 1 | - | x | - | x |
|
|
// | 1i | - | - | - | - |
|
|
// | 2 | - | - | - | x |
|
|
// --------+--------+--------+--------+--------+--------+
|
|
return { result: rs, logData: lds };
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
// https://github.com/gorhill/uBlock/issues/3208
|
|
// Mind case insensitivity.
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/1147
|
|
// Do not strict-block if the filter pattern does not contain at least one
|
|
// token character.
|
|
|
|
const validateStrictBlock = function(fctxt, logData) {
|
|
if ( typeof logData.regex !== 'string' ) { return false; }
|
|
if ( typeof logData.raw === 'string' && /\w/.test(logData.raw) === false ) {
|
|
return false;
|
|
}
|
|
const url = fctxt.url;
|
|
const re = new RegExp(logData.regex, 'i');
|
|
const match = re.exec(url.toLowerCase());
|
|
if ( match === null ) { return false; }
|
|
|
|
// https://github.com/chrisaljoudi/uBlock/issues/1128
|
|
// https://github.com/chrisaljoudi/uBlock/issues/1212
|
|
// Verify that the end of the match is anchored to the end of the
|
|
// hostname.
|
|
// https://github.com/uBlockOrigin/uAssets/issues/7619#issuecomment-653010310
|
|
// Also match FQDN.
|
|
const hostname = fctxt.getHostname();
|
|
const hnpos = url.indexOf(hostname);
|
|
const hnlen = hostname.length;
|
|
const end = match.index + match[0].length - hnpos - hnlen;
|
|
return end === 0 || end === 1 ||
|
|
end === 2 && url.charCodeAt(hnpos + hnlen) === 0x2E /* '.' */;
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
// Intercept and filter behind-the-scene requests.
|
|
|
|
const onBeforeBehindTheSceneRequest = function(fctxt) {
|
|
const pageStore = µb.pageStoreFromTabId(fctxt.tabId);
|
|
if ( pageStore === null ) { return; }
|
|
|
|
// https://github.com/gorhill/uBlock/issues/3150
|
|
// Ability to globally block CSP reports MUST also apply to
|
|
// behind-the-scene network requests.
|
|
|
|
let result = 0;
|
|
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/339
|
|
// Need to also test against `-scheme` since tabOrigin is normalized.
|
|
// Not especially elegant but for now this accomplishes the purpose of
|
|
// not dealing with network requests fired from a synthetic scope,
|
|
// that is unless advanced user mode is enabled.
|
|
|
|
if (
|
|
fctxt.tabOrigin.endsWith('-scheme') === false &&
|
|
isNetworkURI(fctxt.tabOrigin) ||
|
|
µb.userSettings.advancedUserEnabled ||
|
|
fctxt.itype === fctxt.CSP_REPORT
|
|
) {
|
|
result = pageStore.filterRequest(fctxt);
|
|
|
|
// The "any-tab" scope is not whitelist-able, and in such case we must
|
|
// use the origin URL as the scope. Most such requests aren't going to
|
|
// be blocked, so we test for whitelisting and modify the result only
|
|
// when the request is being blocked.
|
|
//
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/1478
|
|
// Also remove potential redirection when request is to be
|
|
// whitelisted.
|
|
if (
|
|
result === 1 &&
|
|
µb.getNetFilteringSwitch(fctxt.tabOrigin) === false
|
|
) {
|
|
result = 2;
|
|
fctxt.redirectURL = undefined;
|
|
fctxt.filter = { engine: 'u', result: 2, raw: 'whitelisted' };
|
|
}
|
|
}
|
|
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/1204
|
|
onBeforeBehindTheSceneRequest.journalAddRequest(fctxt, result);
|
|
|
|
if ( logger.enabled ) {
|
|
fctxt.setRealm('network').toLogger();
|
|
}
|
|
|
|
// Redirected
|
|
|
|
if ( fctxt.redirectURL !== undefined ) {
|
|
return { redirectUrl: patchLocalRedirectURL(fctxt.redirectURL) };
|
|
}
|
|
|
|
// Blocked?
|
|
|
|
if ( result === 1 ) {
|
|
return { cancel: true };
|
|
}
|
|
};
|
|
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/1204
|
|
// Report the tabless network requests to all page stores matching the
|
|
// document origin. This is an approximation, there is unfortunately no
|
|
// way to know for sure which exact page triggered a tabless network
|
|
// request.
|
|
|
|
{
|
|
const pageStores = new Set();
|
|
let hostname = '';
|
|
let pageStoresToken = 0;
|
|
|
|
const reset = function() {
|
|
hostname = '';
|
|
pageStores.clear();
|
|
pageStoresToken = 0;
|
|
};
|
|
|
|
const gc = ( ) => {
|
|
if ( pageStoresToken !== µb.pageStoresToken ) { return reset(); }
|
|
gcTimer.on(30011);
|
|
};
|
|
|
|
const gcTimer = vAPI.defer.create(gc);
|
|
|
|
onBeforeBehindTheSceneRequest.journalAddRequest = (fctxt, result) => {
|
|
const docHostname = fctxt.getDocHostname();
|
|
if (
|
|
docHostname !== hostname ||
|
|
pageStoresToken !== µb.pageStoresToken
|
|
) {
|
|
hostname = docHostname;
|
|
pageStores.clear();
|
|
for ( const pageStore of µb.pageStores.values() ) {
|
|
if ( pageStore.tabHostname !== docHostname ) { continue; }
|
|
pageStores.add(pageStore);
|
|
}
|
|
pageStoresToken = µb.pageStoresToken;
|
|
gcTimer.offon(30011);
|
|
}
|
|
for ( const pageStore of pageStores ) {
|
|
pageStore.journalAddRequest(fctxt, result);
|
|
}
|
|
};
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
// To handle:
|
|
// - Media elements larger than n kB
|
|
// - Scriptlet injection (requires ability to modify response body)
|
|
// - HTML filtering (requires ability to modify response body)
|
|
// - CSP injection
|
|
|
|
const onHeadersReceived = function(details) {
|
|
|
|
const fctxt = µb.filteringContext.fromWebrequestDetails(details);
|
|
const isRootDoc = fctxt.itype === fctxt.MAIN_FRAME;
|
|
|
|
let pageStore = µb.pageStoreFromTabId(fctxt.tabId);
|
|
if ( pageStore === null ) {
|
|
if ( isRootDoc === false ) { return; }
|
|
pageStore = µb.bindTabToPageStore(fctxt.tabId, 'beforeRequest');
|
|
}
|
|
if ( pageStore.getNetFilteringSwitch(fctxt) === false ) { return; }
|
|
|
|
if ( fctxt.itype === fctxt.IMAGE || fctxt.itype === fctxt.MEDIA ) {
|
|
const result = foilLargeMediaElement(details, fctxt, pageStore);
|
|
if ( result !== undefined ) { return result; }
|
|
}
|
|
|
|
// Keep in mind response headers will be modified in-place if needed, so
|
|
// `details.responseHeaders` will always point to the modified response
|
|
// headers.
|
|
const { responseHeaders } = details;
|
|
if ( Array.isArray(responseHeaders) === false ) { return; }
|
|
|
|
if ( isRootDoc === false ) {
|
|
const result = pageStore.filterOnHeaders(fctxt, responseHeaders);
|
|
if ( result !== 0 ) {
|
|
if ( logger.enabled ) {
|
|
fctxt.setRealm('network').toLogger();
|
|
}
|
|
if ( result === 1 ) {
|
|
pageStore.journalAddRequest(fctxt, 1);
|
|
return { cancel: true };
|
|
}
|
|
}
|
|
}
|
|
|
|
const mime = mimeFromHeaders(responseHeaders);
|
|
|
|
// https://github.com/gorhill/uBlock/issues/2813
|
|
// Disable the blocking of large media elements if the document is itself
|
|
// a media element: the resource was not prevented from loading so no
|
|
// point to further block large media elements for the current document.
|
|
if ( isRootDoc ) {
|
|
if ( reMediaContentTypes.test(mime) ) {
|
|
pageStore.allowLargeMediaElementsUntil = 0;
|
|
// Fall-through: this could be an SVG document, which supports
|
|
// script tags.
|
|
}
|
|
}
|
|
|
|
if ( bodyFilterer.canFilter(fctxt, details) ) {
|
|
const jobs = [];
|
|
// `replace=` filter option
|
|
const replaceDirectives =
|
|
staticNetFilteringEngine.matchAndFetchModifiers(fctxt, 'replace');
|
|
if ( replaceDirectives ) {
|
|
jobs.push({
|
|
fn: textResponseFilterer,
|
|
args: [ replaceDirectives ],
|
|
});
|
|
}
|
|
// html filtering
|
|
if ( mime === 'text/html' || mime === 'application/xhtml+xml' ) {
|
|
const selectors = htmlFilteringEngine.retrieve(fctxt);
|
|
if ( selectors ) {
|
|
jobs.push({
|
|
fn: htmlResponseFilterer,
|
|
args: [ selectors ],
|
|
});
|
|
}
|
|
}
|
|
if ( jobs.length !== 0 ) {
|
|
bodyFilterer.doFilter(details.requestId, fctxt, jobs);
|
|
}
|
|
}
|
|
|
|
let modifiedHeaders = false;
|
|
if ( httpheaderFilteringEngine.apply(fctxt, responseHeaders) === true ) {
|
|
modifiedHeaders = true;
|
|
}
|
|
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/229#issuecomment-2220354261
|
|
// Inject CSP/PP in document resource only
|
|
if ( fctxt.isDocument() ) {
|
|
if ( injectCSP(fctxt, pageStore, responseHeaders) === true ) {
|
|
modifiedHeaders = true;
|
|
}
|
|
if ( injectPP(fctxt, pageStore, responseHeaders) === true ) {
|
|
modifiedHeaders = true;
|
|
}
|
|
}
|
|
|
|
// https://bugzilla.mozilla.org/show_bug.cgi?id=1376932
|
|
// Prevent document from being cached by the browser if we modified it,
|
|
// either through HTML filtering and/or modified response headers.
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/229
|
|
// Use `no-cache` instead of `no-cache, no-store, must-revalidate`, this
|
|
// allows Firefox's offline mode to work as expected.
|
|
if ( modifiedHeaders && dontCacheResponseHeaders ) {
|
|
const cacheControl = µb.hiddenSettings.cacheControlForFirefox1376932;
|
|
if ( cacheControl !== 'unset' ) {
|
|
let i = headerIndexFromName('cache-control', responseHeaders);
|
|
if ( i !== -1 ) {
|
|
responseHeaders[i].value = cacheControl;
|
|
} else {
|
|
responseHeaders.push({ name: 'Cache-Control', value: cacheControl });
|
|
}
|
|
modifiedHeaders = true;
|
|
}
|
|
}
|
|
|
|
if ( modifiedHeaders ) {
|
|
return { responseHeaders };
|
|
}
|
|
};
|
|
|
|
const reMediaContentTypes = /^(?:audio|image|video)\/|(?:\/ogg)$/;
|
|
|
|
/******************************************************************************/
|
|
|
|
const mimeFromHeaders = headers => {
|
|
if ( Array.isArray(headers) === false ) { return ''; }
|
|
return mimeFromContentType(headerValueFromName('content-type', headers));
|
|
};
|
|
|
|
const mimeFromContentType = contentType => {
|
|
const match = reContentTypeMime.exec(contentType);
|
|
if ( match === null ) { return ''; }
|
|
return match[0].toLowerCase();
|
|
};
|
|
|
|
const reContentTypeMime = /^[^;]+/i;
|
|
|
|
/******************************************************************************/
|
|
|
|
function textResponseFilterer(session, directives) {
|
|
const applied = [];
|
|
for ( const directive of directives ) {
|
|
if ( directive.refs instanceof Object === false ) { continue; }
|
|
if ( directive.result !== 1 ) {
|
|
applied.push(directive);
|
|
continue;
|
|
}
|
|
const { refs } = directive;
|
|
if ( refs.$cache === null ) {
|
|
refs.$cache = sfp.parseReplaceValue(refs.value);
|
|
}
|
|
const cache = refs.$cache;
|
|
if ( cache === undefined ) { continue; }
|
|
cache.re.lastIndex = 0;
|
|
if ( cache.re.test(session.getString()) !== true ) { continue; }
|
|
cache.re.lastIndex = 0;
|
|
session.setString(session.getString().replace(
|
|
cache.re,
|
|
cache.replacement
|
|
));
|
|
applied.push(directive);
|
|
}
|
|
if ( applied.length === 0 ) { return; }
|
|
if ( logger.enabled !== true ) { return; }
|
|
session.setRealm('network')
|
|
.pushFilters(applied.map(a => a.logData()))
|
|
.toLogger();
|
|
}
|
|
|
|
/******************************************************************************/
|
|
|
|
function htmlResponseFilterer(session, selectors) {
|
|
if ( htmlResponseFilterer.domParser === null ) {
|
|
htmlResponseFilterer.domParser = new DOMParser();
|
|
htmlResponseFilterer.xmlSerializer = new XMLSerializer();
|
|
}
|
|
|
|
const doc = htmlResponseFilterer.domParser.parseFromString(
|
|
session.getString(),
|
|
session.mime
|
|
);
|
|
|
|
if ( selectors === undefined ) { return; }
|
|
if ( htmlFilteringEngine.apply(doc, session, selectors) !== true ) { return; }
|
|
|
|
// https://stackoverflow.com/questions/6088972/get-doctype-of-an-html-as-string-with-javascript/10162353#10162353
|
|
const doctypeStr = [
|
|
doc.doctype instanceof Object ?
|
|
htmlResponseFilterer.xmlSerializer.serializeToString(doc.doctype) + '\n' :
|
|
'',
|
|
doc.documentElement.outerHTML,
|
|
].join('\n');
|
|
session.setString(doctypeStr);
|
|
}
|
|
htmlResponseFilterer.domParser = null;
|
|
htmlResponseFilterer.xmlSerializer = null;
|
|
|
|
|
|
/*******************************************************************************
|
|
|
|
The response body filterer is responsible for:
|
|
|
|
- Realize static network filter option `replace=`
|
|
- HTML filtering
|
|
|
|
**/
|
|
|
|
const bodyFilterer = (( ) => {
|
|
const sessions = new Map();
|
|
const reContentTypeCharset = /charset=['"]?([^'" ]+)/i;
|
|
const otherValidMimes = new Set([
|
|
'application/javascript',
|
|
'application/json',
|
|
'application/mpegurl',
|
|
'application/vnd.api+json',
|
|
'application/vnd.apple.mpegurl',
|
|
'application/vnd.apple.mpegurl.audio',
|
|
'application/x-javascript',
|
|
'application/x-mpegurl',
|
|
'application/xhtml+xml',
|
|
'application/xml',
|
|
'audio/mpegurl',
|
|
'audio/x-mpegurl',
|
|
]);
|
|
const BINARY_TYPES = fc.FONT | fc.IMAGE | fc.MEDIA | fc.WEBSOCKET;
|
|
const MAX_RESPONSE_BUFFER_LENGTH = 10 * 1024 * 1024;
|
|
|
|
let textDecoder, textEncoder;
|
|
let mime = '';
|
|
let charset = '';
|
|
|
|
const contentTypeFromDetails = details => {
|
|
switch ( details.type ) {
|
|
case 'script':
|
|
return 'text/javascript; charset=utf-8';
|
|
case 'stylesheet':
|
|
return 'text/css';
|
|
default:
|
|
break;
|
|
}
|
|
return '';
|
|
};
|
|
|
|
const charsetFromContentType = contentType => {
|
|
const match = reContentTypeCharset.exec(contentType);
|
|
if ( match === null ) { return; }
|
|
return match[1].toLowerCase();
|
|
};
|
|
|
|
const charsetFromMime = mime => {
|
|
switch ( mime ) {
|
|
case 'application/xml':
|
|
case 'application/xhtml+xml':
|
|
case 'text/html':
|
|
case 'text/css':
|
|
return;
|
|
default:
|
|
break;
|
|
}
|
|
return 'utf-8';
|
|
};
|
|
|
|
const charsetFromStream = bytes => {
|
|
if ( bytes.length < 3 ) { return; }
|
|
if ( bytes[0] === 0xEF && bytes[1] === 0xBB && bytes[2] === 0xBF ) {
|
|
return 'utf-8';
|
|
}
|
|
let i = -1;
|
|
while ( i < 65536 ) {
|
|
i += 1;
|
|
/* c */ if ( bytes[i+0] !== 0x63 ) { continue; }
|
|
/* h */ if ( bytes[i+1] !== 0x68 ) { continue; }
|
|
/* a */ if ( bytes[i+2] !== 0x61 ) { continue; }
|
|
/* r */ if ( bytes[i+3] !== 0x72 ) { continue; }
|
|
/* s */ if ( bytes[i+4] !== 0x73 ) { continue; }
|
|
/* e */ if ( bytes[i+5] !== 0x65 ) { continue; }
|
|
/* t */ if ( bytes[i+6] !== 0x74 ) { continue; }
|
|
break;
|
|
}
|
|
if ( (i + 40) >= 65536 ) { return; }
|
|
i += 8;
|
|
// find first alpha character
|
|
let j = -1;
|
|
while ( j < 8 ) {
|
|
j += 1;
|
|
const c = bytes[i+j];
|
|
if ( c >= 0x41 && c <= 0x5A ) { break; }
|
|
if ( c >= 0x61 && c <= 0x7A ) { break; }
|
|
}
|
|
if ( j === 8 ) { return; }
|
|
i += j;
|
|
// Collect characters until first non charset-name-character
|
|
const chars = [];
|
|
j = 0;
|
|
while ( j < 24 ) {
|
|
const c = bytes[i+j];
|
|
if ( c < 0x2D ) { break; }
|
|
if ( c > 0x2D && c < 0x30 ) { break; }
|
|
if ( c > 0x39 && c < 0x41 ) { break; }
|
|
if ( c > 0x5A && c < 0x61 ) { break; }
|
|
if ( c > 0x7A ) { break; }
|
|
chars.push(c);
|
|
j += 1;
|
|
}
|
|
if ( j === 20 ) { return; }
|
|
return String.fromCharCode(...chars).toLowerCase();
|
|
};
|
|
|
|
const streamClose = (session, buffer) => {
|
|
if ( buffer !== undefined ) {
|
|
session.stream.write(buffer);
|
|
} else if ( session.buffer !== undefined ) {
|
|
session.stream.write(session.buffer);
|
|
}
|
|
session.stream.close();
|
|
};
|
|
|
|
const onStreamData = function(ev) {
|
|
const session = sessions.get(this);
|
|
if ( session === undefined ) {
|
|
this.write(ev.data);
|
|
this.disconnect();
|
|
return;
|
|
}
|
|
if ( this.status !== 'transferringdata' ) {
|
|
if ( this.status !== 'finishedtransferringdata' ) {
|
|
sessions.delete(this);
|
|
this.disconnect();
|
|
return;
|
|
}
|
|
}
|
|
if ( session.buffer === null ) {
|
|
session.buffer = new Uint8Array(ev.data);
|
|
return;
|
|
}
|
|
const buffer = new Uint8Array(
|
|
session.buffer.byteLength + ev.data.byteLength
|
|
);
|
|
buffer.set(session.buffer);
|
|
buffer.set(new Uint8Array(ev.data), session.buffer.byteLength);
|
|
session.buffer = buffer;
|
|
if ( session.buffer.length >= MAX_RESPONSE_BUFFER_LENGTH ) {
|
|
sessions.delete(this);
|
|
this.write(session.buffer);
|
|
this.disconnect();
|
|
}
|
|
};
|
|
|
|
const onStreamStop = function() {
|
|
const session = sessions.get(this);
|
|
sessions.delete(this);
|
|
if ( session === undefined || session.buffer === null ) {
|
|
this.close();
|
|
return;
|
|
}
|
|
if ( this.status !== 'finishedtransferringdata' ) { return; }
|
|
|
|
// If encoding is still unknown, try to extract from stream data.
|
|
// Just assume utf-8 if ultimately no encoding can be looked up.
|
|
if ( session.charset === undefined ) {
|
|
const charsetFound = charsetFromStream(session.buffer);
|
|
if ( charsetFound !== undefined ) {
|
|
const charsetUsed = textEncode.normalizeCharset(charsetFound);
|
|
if ( charsetUsed === undefined ) { return streamClose(session); }
|
|
session.charset = charsetUsed;
|
|
} else {
|
|
session.charset = 'utf-8';
|
|
}
|
|
}
|
|
|
|
while ( session.jobs.length !== 0 ) {
|
|
const job = session.jobs.shift();
|
|
job.fn(session, ...job.args);
|
|
}
|
|
if ( session.modified !== true ) { return streamClose(session); }
|
|
|
|
if ( textEncoder === undefined ) {
|
|
textEncoder = new TextEncoder();
|
|
}
|
|
let encodedStream = textEncoder.encode(session.str);
|
|
|
|
if ( session.charset !== 'utf-8' ) {
|
|
encodedStream = textEncode.encode(session.charset, encodedStream);
|
|
}
|
|
|
|
streamClose(session, encodedStream);
|
|
};
|
|
|
|
const onStreamError = function() {
|
|
sessions.delete(this);
|
|
};
|
|
|
|
return class Session extends µb.FilteringContext {
|
|
constructor(fctxt, mime, charset, jobs) {
|
|
super(fctxt);
|
|
this.stream = null;
|
|
this.buffer = null;
|
|
this.mime = mime;
|
|
this.charset = charset;
|
|
this.str = null;
|
|
this.modified = false;
|
|
this.jobs = jobs;
|
|
}
|
|
getString() {
|
|
if ( this.str !== null ) { return this.str; }
|
|
if ( textDecoder !== undefined ) {
|
|
if ( textDecoder.encoding !== this.charset ) {
|
|
textDecoder = undefined;
|
|
}
|
|
}
|
|
if ( textDecoder === undefined ) {
|
|
textDecoder = new TextDecoder(this.charset);
|
|
}
|
|
this.str = textDecoder.decode(this.buffer);
|
|
return this.str;
|
|
}
|
|
setString(s) {
|
|
this.str = s;
|
|
this.modified = true;
|
|
}
|
|
static doFilter(requestId, fctxt, jobs) {
|
|
if ( jobs.length === 0 ) { return; }
|
|
const session = new Session(fctxt, mime, charset, jobs);
|
|
session.stream = browser.webRequest.filterResponseData(requestId);
|
|
session.stream.ondata = onStreamData;
|
|
session.stream.onstop = onStreamStop;
|
|
session.stream.onerror = onStreamError;
|
|
sessions.set(session.stream, session);
|
|
}
|
|
static canFilter(fctxt, details) {
|
|
if ( µb.canFilterResponseData !== true ) { return; }
|
|
|
|
if ( (fctxt.itype & BINARY_TYPES) !== 0 ) { return; }
|
|
|
|
if ( fctxt.method !== fc.METHOD_GET ) {
|
|
if ( fctxt.method !== fc.METHOD_POST ) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
// https://github.com/gorhill/uBlock/issues/3478
|
|
const statusCode = details.statusCode || 0;
|
|
if ( statusCode === 0 ) { return; }
|
|
|
|
const hostname = fctxt.getHostname();
|
|
if ( hostname === '' ) { return; }
|
|
|
|
// https://bugzilla.mozilla.org/show_bug.cgi?id=1426789
|
|
const headers = details.responseHeaders;
|
|
const disposition = headerValueFromName('content-disposition', headers);
|
|
if ( disposition !== '' ) {
|
|
if ( disposition.startsWith('inline') === false ) { return; }
|
|
}
|
|
|
|
mime = 'text/plain';
|
|
charset = 'utf-8';
|
|
const contentType = headerValueFromName('content-type', headers) ||
|
|
contentTypeFromDetails(details);
|
|
if ( contentType !== '' ) {
|
|
mime = mimeFromContentType(contentType);
|
|
if ( mime === undefined ) { return; }
|
|
if ( mime.startsWith('text/') === false ) {
|
|
if ( otherValidMimes.has(mime) === false ) { return; }
|
|
}
|
|
charset = charsetFromContentType(contentType);
|
|
if ( charset !== undefined ) {
|
|
charset = textEncode.normalizeCharset(charset);
|
|
if ( charset === undefined ) { return; }
|
|
} else {
|
|
charset = charsetFromMime(mime);
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
};
|
|
})();
|
|
|
|
/******************************************************************************/
|
|
|
|
const injectCSP = function(fctxt, pageStore, responseHeaders) {
|
|
const cspSubsets = [];
|
|
const requestType = fctxt.type;
|
|
|
|
// Start collecting policies >>>>>>>>
|
|
|
|
// ======== built-in policies
|
|
|
|
const builtinDirectives = [];
|
|
|
|
if ( pageStore.filterScripting(fctxt, true) === 1 ) {
|
|
builtinDirectives.push(µb.cspNoScripting);
|
|
if ( logger.enabled ) {
|
|
fctxt.setRealm('network').setType('scripting').toLogger();
|
|
}
|
|
}
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/422
|
|
// We need to derive a special context for filtering `inline-script`,
|
|
// as the embedding document for this "resource" will always be the
|
|
// frame itself, not that of the parent of the frame.
|
|
else {
|
|
const fctxt2 = fctxt.duplicate();
|
|
fctxt2.type = 'inline-script';
|
|
fctxt2.setDocOriginFromURL(fctxt.url);
|
|
const result = pageStore.filterRequest(fctxt2);
|
|
if ( result === 1 ) {
|
|
builtinDirectives.push(µb.cspNoInlineScript);
|
|
}
|
|
if ( result === 2 && logger.enabled ) {
|
|
fctxt2.setRealm('network').toLogger();
|
|
}
|
|
}
|
|
|
|
// https://github.com/gorhill/uBlock/issues/1539
|
|
// - Use a CSP to also forbid inline fonts if remote fonts are blocked.
|
|
fctxt.type = 'inline-font';
|
|
if ( pageStore.filterRequest(fctxt) === 1 ) {
|
|
builtinDirectives.push(µb.cspNoInlineFont);
|
|
if ( logger.enabled ) {
|
|
fctxt.setRealm('network').toLogger();
|
|
}
|
|
}
|
|
|
|
if ( builtinDirectives.length !== 0 ) {
|
|
cspSubsets[0] = builtinDirectives.join(', ');
|
|
}
|
|
|
|
// ======== filter-based policies
|
|
|
|
// Static filtering.
|
|
|
|
fctxt.type = requestType;
|
|
const staticDirectives =
|
|
staticNetFilteringEngine.matchAndFetchModifiers(fctxt, 'csp');
|
|
if ( staticDirectives !== undefined ) {
|
|
for ( const directive of staticDirectives ) {
|
|
if ( directive.result !== 1 ) { continue; }
|
|
cspSubsets.push(directive.value);
|
|
}
|
|
}
|
|
|
|
// URL filtering `allow` rules override static filtering.
|
|
if (
|
|
cspSubsets.length !== 0 &&
|
|
sessionURLFiltering.evaluateZ(
|
|
fctxt.getTabHostname(),
|
|
fctxt.url,
|
|
'csp'
|
|
) === 2
|
|
) {
|
|
if ( logger.enabled ) {
|
|
fctxt.setRealm('network')
|
|
.setType('csp')
|
|
.setFilter(sessionURLFiltering.toLogData())
|
|
.toLogger();
|
|
}
|
|
return;
|
|
}
|
|
|
|
// Dynamic filtering `allow` rules override static filtering.
|
|
if (
|
|
cspSubsets.length !== 0 &&
|
|
µb.userSettings.advancedUserEnabled &&
|
|
sessionFirewall.evaluateCellZY(
|
|
fctxt.getTabHostname(),
|
|
fctxt.getTabHostname(),
|
|
'*'
|
|
) === 2
|
|
) {
|
|
if ( logger.enabled ) {
|
|
fctxt.setRealm('network')
|
|
.setType('csp')
|
|
.setFilter(sessionFirewall.toLogData())
|
|
.toLogger();
|
|
}
|
|
return;
|
|
}
|
|
|
|
// <<<<<<<< All policies have been collected
|
|
|
|
// Static CSP policies will be applied.
|
|
|
|
if ( logger.enabled && staticDirectives !== undefined ) {
|
|
fctxt.setRealm('network')
|
|
.pushFilters(staticDirectives.map(a => a.logData()))
|
|
.toLogger();
|
|
}
|
|
|
|
if ( cspSubsets.length === 0 ) { return; }
|
|
|
|
µb.updateToolbarIcon(fctxt.tabId, 0b0010);
|
|
|
|
// Use comma to merge CSP directives.
|
|
// Ref.: https://www.w3.org/TR/CSP2/#implementation-considerations
|
|
//
|
|
// https://github.com/gorhill/uMatrix/issues/967
|
|
// Inject a new CSP header rather than modify an existing one, except
|
|
// if the current environment does not support merging headers:
|
|
// Firefox 58/webext and less can't merge CSP headers, so we will merge
|
|
// them here.
|
|
|
|
responseHeaders.push({
|
|
name: 'Content-Security-Policy',
|
|
value: cspSubsets.join(', ')
|
|
});
|
|
|
|
return true;
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
const injectPP = function(fctxt, pageStore, responseHeaders) {
|
|
const permissions = [];
|
|
const directives = staticNetFilteringEngine.matchAndFetchModifiers(fctxt, 'permissions');
|
|
if ( directives !== undefined ) {
|
|
for ( const directive of directives ) {
|
|
if ( directive.result !== 1 ) { continue; }
|
|
permissions.push(directive.value.replace('|', ', '));
|
|
}
|
|
}
|
|
|
|
if ( logger.enabled && directives !== undefined ) {
|
|
fctxt.setRealm('network')
|
|
.pushFilters(directives.map(a => a.logData()))
|
|
.toLogger();
|
|
}
|
|
|
|
if ( permissions.length === 0 ) { return; }
|
|
|
|
µb.updateToolbarIcon(fctxt.tabId, 0x02);
|
|
|
|
responseHeaders.push({
|
|
name: 'permissions-policy',
|
|
value: permissions.join(', ')
|
|
});
|
|
|
|
return true;
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
// https://github.com/gorhill/uBlock/issues/1163
|
|
// "Block elements by size".
|
|
// https://github.com/gorhill/uBlock/issues/1390#issuecomment-187310719
|
|
// Do not foil when the media element is fetched from the browser
|
|
// cache. This works only when the webext API supports the `fromCache`
|
|
// property (Firefox).
|
|
|
|
const foilLargeMediaElement = function(details, fctxt, pageStore) {
|
|
if ( details.fromCache === true ) { return; }
|
|
|
|
let size = 0;
|
|
if ( µb.userSettings.largeMediaSize !== 0 ) {
|
|
const headers = details.responseHeaders;
|
|
const i = headerIndexFromName('content-length', headers);
|
|
if ( i === -1 ) { return; }
|
|
size = parseInt(headers[i].value, 10) || 0;
|
|
}
|
|
|
|
const result = pageStore.filterLargeMediaElement(fctxt, size);
|
|
if ( result === 0 ) { return; }
|
|
|
|
if ( logger.enabled ) {
|
|
fctxt.setRealm('network').toLogger();
|
|
}
|
|
|
|
return { cancel: true };
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
// Caller must ensure headerName is normalized to lower case.
|
|
|
|
const headerIndexFromName = function(headerName, headers) {
|
|
let i = headers.length;
|
|
while ( i-- ) {
|
|
if ( headers[i].name.toLowerCase() === headerName ) {
|
|
return i;
|
|
}
|
|
}
|
|
return -1;
|
|
};
|
|
|
|
const headerValueFromName = function(headerName, headers) {
|
|
const i = headerIndexFromName(headerName, headers);
|
|
return i !== -1 ? headers[i].value : '';
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
const strictBlockBypasser = {
|
|
hostnameToDeadlineMap: new Map(),
|
|
cleanupTimer: vAPI.defer.create(( ) => {
|
|
strictBlockBypasser.cleanup();
|
|
}),
|
|
|
|
cleanup: function() {
|
|
for ( const [ hostname, deadline ] of this.hostnameToDeadlineMap ) {
|
|
if ( deadline <= Date.now() ) {
|
|
this.hostnameToDeadlineMap.delete(hostname);
|
|
}
|
|
}
|
|
},
|
|
|
|
revokeTime: function() {
|
|
return Date.now() + µb.hiddenSettings.strictBlockingBypassDuration * 1000;
|
|
},
|
|
|
|
bypass: function(hostname) {
|
|
if ( typeof hostname !== 'string' || hostname === '' ) { return; }
|
|
this.hostnameToDeadlineMap.set(hostname, this.revokeTime());
|
|
},
|
|
|
|
isBypassed: function(hostname) {
|
|
if ( this.hostnameToDeadlineMap.size === 0 ) { return false; }
|
|
this.cleanupTimer.on({ sec: µb.hiddenSettings.strictBlockingBypassDuration + 10 });
|
|
for (;;) {
|
|
const deadline = this.hostnameToDeadlineMap.get(hostname);
|
|
if ( deadline !== undefined ) {
|
|
if ( deadline > Date.now() ) {
|
|
this.hostnameToDeadlineMap.set(hostname, this.revokeTime());
|
|
return true;
|
|
}
|
|
this.hostnameToDeadlineMap.delete(hostname);
|
|
}
|
|
const pos = hostname.indexOf('.');
|
|
if ( pos === -1 ) { break; }
|
|
hostname = hostname.slice(pos + 1);
|
|
}
|
|
return false;
|
|
}
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/2350
|
|
// Added scriptlet injection attempt at onResponseStarted time as per
|
|
// https://github.com/AdguardTeam/AdguardBrowserExtension/issues/1029 and
|
|
// https://github.com/AdguardTeam/AdguardBrowserExtension/blob/9ab85be5/Extension/src/background/webrequest.js#L620
|
|
|
|
const webRequest = {
|
|
onBeforeRequest,
|
|
|
|
start: (( ) => {
|
|
vAPI.net = new vAPI.Net();
|
|
if ( vAPI.Net.canSuspend() ) {
|
|
vAPI.net.suspend();
|
|
}
|
|
|
|
return ( ) => {
|
|
vAPI.net.setSuspendableListener(onBeforeRequest);
|
|
vAPI.net.addListener(
|
|
'onHeadersReceived',
|
|
onHeadersReceived,
|
|
{ urls: [ 'http://*/*', 'https://*/*' ] },
|
|
[ 'blocking', 'responseHeaders' ]
|
|
);
|
|
vAPI.net.addListener(
|
|
'onResponseStarted',
|
|
details => {
|
|
if ( details.tabId === -1 ) { return; }
|
|
const pageStore = µb.pageStoreFromTabId(details.tabId);
|
|
if ( pageStore === null ) { return; }
|
|
if ( pageStore.getNetFilteringSwitch() === false ) { return; }
|
|
scriptletFilteringEngine.injectNow(details);
|
|
},
|
|
{
|
|
types: [ 'main_frame', 'sub_frame' ],
|
|
urls: [ 'http://*/*', 'https://*/*' ]
|
|
}
|
|
);
|
|
vAPI.defer.once({ sec: µb.hiddenSettings.toolbarWarningTimeout }).then(( ) => {
|
|
if ( vAPI.net.hasUnprocessedRequest() === false ) { return; }
|
|
vAPI.net.removeUnprocessedRequest();
|
|
return vAPI.tabs.getCurrent();
|
|
}).then(tab => {
|
|
if ( tab instanceof Object === false ) { return; }
|
|
µb.updateToolbarIcon(tab.id, 0b0110);
|
|
});
|
|
vAPI.net.unsuspend({ all: true });
|
|
};
|
|
})(),
|
|
|
|
strictBlockBypass: hostname => {
|
|
strictBlockBypasser.bypass(hostname);
|
|
},
|
|
};
|
|
|
|
/******************************************************************************/
|
|
|
|
export default webRequest;
|
|
|
|
/******************************************************************************/
|