🎉 initiate project *astro_rewrite*
This commit is contained in:
sindrekjelsrud
parent
ffd4d5e86c
commit
2ba37bfbe3
8658 changed files with 2268794 additions and 2538 deletions
98
node_modules/sitemap/dist/lib/errors.d.ts
generated
vendored
Normal file
98
node_modules/sitemap/dist/lib/errors.d.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
/*!
|
||||
* Sitemap
|
||||
* Copyright(c) 2011 Eugene Kalinin
|
||||
* MIT Licensed
|
||||
*/
|
||||
/**
|
||||
* URL in SitemapItem does not exist
|
||||
*/
|
||||
export declare class NoURLError extends Error {
|
||||
constructor(message?: string);
|
||||
}
|
||||
/**
|
||||
* Config was not passed to SitemapItem constructor
|
||||
*/
|
||||
export declare class NoConfigError extends Error {
|
||||
constructor(message?: string);
|
||||
}
|
||||
/**
|
||||
* changefreq property in sitemap is invalid
|
||||
*/
|
||||
export declare class ChangeFreqInvalidError extends Error {
|
||||
constructor(url: string, changefreq: any);
|
||||
}
|
||||
/**
|
||||
* priority property in sitemap is invalid
|
||||
*/
|
||||
export declare class PriorityInvalidError extends Error {
|
||||
constructor(url: string, priority: any);
|
||||
}
|
||||
/**
|
||||
* SitemapIndex target Folder does not exists
|
||||
*/
|
||||
export declare class UndefinedTargetFolder extends Error {
|
||||
constructor(message?: string);
|
||||
}
|
||||
export declare class InvalidVideoFormat extends Error {
|
||||
constructor(url: string);
|
||||
}
|
||||
export declare class InvalidVideoDuration extends Error {
|
||||
constructor(url: string, duration: any);
|
||||
}
|
||||
export declare class InvalidVideoDescription extends Error {
|
||||
constructor(url: string, length: number);
|
||||
}
|
||||
export declare class InvalidVideoRating extends Error {
|
||||
constructor(url: string, title: any, rating: any);
|
||||
}
|
||||
export declare class InvalidAttrValue extends Error {
|
||||
constructor(key: string, val: any, validator: RegExp);
|
||||
}
|
||||
export declare class InvalidAttr extends Error {
|
||||
constructor(key: string);
|
||||
}
|
||||
export declare class InvalidNewsFormat extends Error {
|
||||
constructor(url: string);
|
||||
}
|
||||
export declare class InvalidNewsAccessValue extends Error {
|
||||
constructor(url: string, access: any);
|
||||
}
|
||||
export declare class XMLLintUnavailable extends Error {
|
||||
constructor(message?: string);
|
||||
}
|
||||
export declare class InvalidVideoTitle extends Error {
|
||||
constructor(url: string, length: number);
|
||||
}
|
||||
export declare class InvalidVideoViewCount extends Error {
|
||||
constructor(url: string, count: number);
|
||||
}
|
||||
export declare class InvalidVideoTagCount extends Error {
|
||||
constructor(url: string, count: number);
|
||||
}
|
||||
export declare class InvalidVideoCategory extends Error {
|
||||
constructor(url: string, count: number);
|
||||
}
|
||||
export declare class InvalidVideoFamilyFriendly extends Error {
|
||||
constructor(url: string, fam: string);
|
||||
}
|
||||
export declare class InvalidVideoRestriction extends Error {
|
||||
constructor(url: string, code: string);
|
||||
}
|
||||
export declare class InvalidVideoRestrictionRelationship extends Error {
|
||||
constructor(url: string, val?: string);
|
||||
}
|
||||
export declare class InvalidVideoPriceType extends Error {
|
||||
constructor(url: string, priceType?: string, price?: string);
|
||||
}
|
||||
export declare class InvalidVideoResolution extends Error {
|
||||
constructor(url: string, resolution: string);
|
||||
}
|
||||
export declare class InvalidVideoPriceCurrency extends Error {
|
||||
constructor(url: string, currency: string);
|
||||
}
|
||||
export declare class EmptyStream extends Error {
|
||||
constructor();
|
||||
}
|
||||
export declare class EmptySitemap extends Error {
|
||||
constructor();
|
||||
}
|
||||
245
node_modules/sitemap/dist/lib/errors.js
generated
vendored
Normal file
245
node_modules/sitemap/dist/lib/errors.js
generated
vendored
Normal file
|
|
@ -0,0 +1,245 @@
|
|||
"use strict";
|
||||
/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
/*!
|
||||
* Sitemap
|
||||
* Copyright(c) 2011 Eugene Kalinin
|
||||
* MIT Licensed
|
||||
*/
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.EmptySitemap = exports.EmptyStream = exports.InvalidVideoPriceCurrency = exports.InvalidVideoResolution = exports.InvalidVideoPriceType = exports.InvalidVideoRestrictionRelationship = exports.InvalidVideoRestriction = exports.InvalidVideoFamilyFriendly = exports.InvalidVideoCategory = exports.InvalidVideoTagCount = exports.InvalidVideoViewCount = exports.InvalidVideoTitle = exports.XMLLintUnavailable = exports.InvalidNewsAccessValue = exports.InvalidNewsFormat = exports.InvalidAttr = exports.InvalidAttrValue = exports.InvalidVideoRating = exports.InvalidVideoDescription = exports.InvalidVideoDuration = exports.InvalidVideoFormat = exports.UndefinedTargetFolder = exports.PriorityInvalidError = exports.ChangeFreqInvalidError = exports.NoConfigError = exports.NoURLError = void 0;
|
||||
/**
|
||||
* URL in SitemapItem does not exist
|
||||
*/
|
||||
class NoURLError extends Error {
|
||||
constructor(message) {
|
||||
super(message || 'URL is required');
|
||||
this.name = 'NoURLError';
|
||||
Error.captureStackTrace(this, NoURLError);
|
||||
}
|
||||
}
|
||||
exports.NoURLError = NoURLError;
|
||||
/**
|
||||
* Config was not passed to SitemapItem constructor
|
||||
*/
|
||||
class NoConfigError extends Error {
|
||||
constructor(message) {
|
||||
super(message || 'SitemapItem requires a configuration');
|
||||
this.name = 'NoConfigError';
|
||||
Error.captureStackTrace(this, NoConfigError);
|
||||
}
|
||||
}
|
||||
exports.NoConfigError = NoConfigError;
|
||||
/**
|
||||
* changefreq property in sitemap is invalid
|
||||
*/
|
||||
class ChangeFreqInvalidError extends Error {
|
||||
constructor(url, changefreq) {
|
||||
super(`${url}: changefreq "${changefreq}" is invalid`);
|
||||
this.name = 'ChangeFreqInvalidError';
|
||||
Error.captureStackTrace(this, ChangeFreqInvalidError);
|
||||
}
|
||||
}
|
||||
exports.ChangeFreqInvalidError = ChangeFreqInvalidError;
|
||||
/**
|
||||
* priority property in sitemap is invalid
|
||||
*/
|
||||
class PriorityInvalidError extends Error {
|
||||
constructor(url, priority) {
|
||||
super(`${url}: priority "${priority}" must be a number between 0 and 1 inclusive`);
|
||||
this.name = 'PriorityInvalidError';
|
||||
Error.captureStackTrace(this, PriorityInvalidError);
|
||||
}
|
||||
}
|
||||
exports.PriorityInvalidError = PriorityInvalidError;
|
||||
/**
|
||||
* SitemapIndex target Folder does not exists
|
||||
*/
|
||||
class UndefinedTargetFolder extends Error {
|
||||
constructor(message) {
|
||||
super(message || 'Target folder must exist');
|
||||
this.name = 'UndefinedTargetFolder';
|
||||
Error.captureStackTrace(this, UndefinedTargetFolder);
|
||||
}
|
||||
}
|
||||
exports.UndefinedTargetFolder = UndefinedTargetFolder;
|
||||
class InvalidVideoFormat extends Error {
|
||||
constructor(url) {
|
||||
super(`${url} video must include thumbnail_loc, title and description fields for videos`);
|
||||
this.name = 'InvalidVideoFormat';
|
||||
Error.captureStackTrace(this, InvalidVideoFormat);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoFormat = InvalidVideoFormat;
|
||||
class InvalidVideoDuration extends Error {
|
||||
constructor(url, duration) {
|
||||
super(`${url} duration "${duration}" must be an integer of seconds between 0 and 28800`);
|
||||
this.name = 'InvalidVideoDuration';
|
||||
Error.captureStackTrace(this, InvalidVideoDuration);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoDuration = InvalidVideoDuration;
|
||||
class InvalidVideoDescription extends Error {
|
||||
constructor(url, length) {
|
||||
const message = `${url}: video description is too long ${length} vs limit of 2048 characters.`;
|
||||
super(message);
|
||||
this.name = 'InvalidVideoDescription';
|
||||
Error.captureStackTrace(this, InvalidVideoDescription);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoDescription = InvalidVideoDescription;
|
||||
class InvalidVideoRating extends Error {
|
||||
constructor(url, title, rating) {
|
||||
super(`${url}: video "${title}" rating "${rating}" must be between 0 and 5 inclusive`);
|
||||
this.name = 'InvalidVideoRating';
|
||||
Error.captureStackTrace(this, InvalidVideoRating);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoRating = InvalidVideoRating;
|
||||
class InvalidAttrValue extends Error {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
constructor(key, val, validator) {
|
||||
super('"' +
|
||||
val +
|
||||
'" tested against: ' +
|
||||
validator +
|
||||
' is not a valid value for attr: "' +
|
||||
key +
|
||||
'"');
|
||||
this.name = 'InvalidAttrValue';
|
||||
Error.captureStackTrace(this, InvalidAttrValue);
|
||||
}
|
||||
}
|
||||
exports.InvalidAttrValue = InvalidAttrValue;
|
||||
// InvalidAttr is only thrown when attrbuilder is called incorrectly internally
|
||||
/* istanbul ignore next */
|
||||
class InvalidAttr extends Error {
|
||||
constructor(key) {
|
||||
super('"' + key + '" is malformed');
|
||||
this.name = 'InvalidAttr';
|
||||
Error.captureStackTrace(this, InvalidAttr);
|
||||
}
|
||||
}
|
||||
exports.InvalidAttr = InvalidAttr;
|
||||
class InvalidNewsFormat extends Error {
|
||||
constructor(url) {
|
||||
super(`${url} News must include publication, publication name, publication language, title, and publication_date for news`);
|
||||
this.name = 'InvalidNewsFormat';
|
||||
Error.captureStackTrace(this, InvalidNewsFormat);
|
||||
}
|
||||
}
|
||||
exports.InvalidNewsFormat = InvalidNewsFormat;
|
||||
class InvalidNewsAccessValue extends Error {
|
||||
constructor(url, access) {
|
||||
super(`${url} News access "${access}" must be either Registration, Subscription or not be present`);
|
||||
this.name = 'InvalidNewsAccessValue';
|
||||
Error.captureStackTrace(this, InvalidNewsAccessValue);
|
||||
}
|
||||
}
|
||||
exports.InvalidNewsAccessValue = InvalidNewsAccessValue;
|
||||
class XMLLintUnavailable extends Error {
|
||||
constructor(message) {
|
||||
super(message || 'xmlLint is not installed. XMLLint is required to validate');
|
||||
this.name = 'XMLLintUnavailable';
|
||||
Error.captureStackTrace(this, XMLLintUnavailable);
|
||||
}
|
||||
}
|
||||
exports.XMLLintUnavailable = XMLLintUnavailable;
|
||||
class InvalidVideoTitle extends Error {
|
||||
constructor(url, length) {
|
||||
super(`${url}: video title is too long ${length} vs 100 character limit`);
|
||||
this.name = 'InvalidVideoTitle';
|
||||
Error.captureStackTrace(this, InvalidVideoTitle);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoTitle = InvalidVideoTitle;
|
||||
class InvalidVideoViewCount extends Error {
|
||||
constructor(url, count) {
|
||||
super(`${url}: video view count must be positive, view count was ${count}`);
|
||||
this.name = 'InvalidVideoViewCount';
|
||||
Error.captureStackTrace(this, InvalidVideoViewCount);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoViewCount = InvalidVideoViewCount;
|
||||
class InvalidVideoTagCount extends Error {
|
||||
constructor(url, count) {
|
||||
super(`${url}: video can have no more than 32 tags, this has ${count}`);
|
||||
this.name = 'InvalidVideoTagCount';
|
||||
Error.captureStackTrace(this, InvalidVideoTagCount);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoTagCount = InvalidVideoTagCount;
|
||||
class InvalidVideoCategory extends Error {
|
||||
constructor(url, count) {
|
||||
super(`${url}: video category can only be 256 characters but was passed ${count}`);
|
||||
this.name = 'InvalidVideoCategory';
|
||||
Error.captureStackTrace(this, InvalidVideoCategory);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoCategory = InvalidVideoCategory;
|
||||
class InvalidVideoFamilyFriendly extends Error {
|
||||
constructor(url, fam) {
|
||||
super(`${url}: video family friendly must be yes or no, was passed "${fam}"`);
|
||||
this.name = 'InvalidVideoFamilyFriendly';
|
||||
Error.captureStackTrace(this, InvalidVideoFamilyFriendly);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoFamilyFriendly = InvalidVideoFamilyFriendly;
|
||||
class InvalidVideoRestriction extends Error {
|
||||
constructor(url, code) {
|
||||
super(`${url}: video restriction must be one or more two letter country codes. Was passed "${code}"`);
|
||||
this.name = 'InvalidVideoRestriction';
|
||||
Error.captureStackTrace(this, InvalidVideoRestriction);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoRestriction = InvalidVideoRestriction;
|
||||
class InvalidVideoRestrictionRelationship extends Error {
|
||||
constructor(url, val) {
|
||||
super(`${url}: video restriction relationship must be either allow or deny. Was passed "${val}"`);
|
||||
this.name = 'InvalidVideoRestrictionRelationship';
|
||||
Error.captureStackTrace(this, InvalidVideoRestrictionRelationship);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoRestrictionRelationship = InvalidVideoRestrictionRelationship;
|
||||
class InvalidVideoPriceType extends Error {
|
||||
constructor(url, priceType, price) {
|
||||
super(priceType === undefined && price === ''
|
||||
? `${url}: video priceType is required when price is not provided`
|
||||
: `${url}: video price type "${priceType}" is not "rent" or "purchase"`);
|
||||
this.name = 'InvalidVideoPriceType';
|
||||
Error.captureStackTrace(this, InvalidVideoPriceType);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoPriceType = InvalidVideoPriceType;
|
||||
class InvalidVideoResolution extends Error {
|
||||
constructor(url, resolution) {
|
||||
super(`${url}: video price resolution "${resolution}" is not hd or sd`);
|
||||
this.name = 'InvalidVideoResolution';
|
||||
Error.captureStackTrace(this, InvalidVideoResolution);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoResolution = InvalidVideoResolution;
|
||||
class InvalidVideoPriceCurrency extends Error {
|
||||
constructor(url, currency) {
|
||||
super(`${url}: video price currency "${currency}" must be a three capital letter abbrieviation for the country currency`);
|
||||
this.name = 'InvalidVideoPriceCurrency';
|
||||
Error.captureStackTrace(this, InvalidVideoPriceCurrency);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoPriceCurrency = InvalidVideoPriceCurrency;
|
||||
class EmptyStream extends Error {
|
||||
constructor() {
|
||||
super('You have ended the stream before anything was written. streamToPromise MUST be called before ending the stream.');
|
||||
this.name = 'EmptyStream';
|
||||
Error.captureStackTrace(this, EmptyStream);
|
||||
}
|
||||
}
|
||||
exports.EmptyStream = EmptyStream;
|
||||
class EmptySitemap extends Error {
|
||||
constructor() {
|
||||
super('You ended the stream without writing anything.');
|
||||
this.name = 'EmptySitemap';
|
||||
Error.captureStackTrace(this, EmptyStream);
|
||||
}
|
||||
}
|
||||
exports.EmptySitemap = EmptySitemap;
|
||||
53
node_modules/sitemap/dist/lib/sitemap-index-parser.d.ts
generated
vendored
Normal file
53
node_modules/sitemap/dist/lib/sitemap-index-parser.d.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
/// <reference types="node" />
|
||||
import { SAXStream } from 'sax';
|
||||
import { Readable, Transform, TransformOptions, TransformCallback } from 'stream';
|
||||
import { IndexItem, ErrorLevel } from './types';
|
||||
declare type Logger = (level: 'warn' | 'error' | 'info' | 'log', ...message: Parameters<Console['log']>[0]) => void;
|
||||
export interface XMLToSitemapIndexItemStreamOptions extends TransformOptions {
|
||||
level?: ErrorLevel;
|
||||
logger?: Logger | false;
|
||||
}
|
||||
/**
|
||||
* Takes a stream of xml and transforms it into a stream of IndexItems
|
||||
* Use this to parse existing sitemap indices into config options compatible with this library
|
||||
*/
|
||||
export declare class XMLToSitemapIndexStream extends Transform {
|
||||
level: ErrorLevel;
|
||||
logger: Logger;
|
||||
saxStream: SAXStream;
|
||||
constructor(opts?: XMLToSitemapIndexItemStreamOptions);
|
||||
_transform(data: string, encoding: string, callback: TransformCallback): void;
|
||||
}
|
||||
/**
|
||||
Read xml and resolve with the configuration that would produce it or reject with
|
||||
an error
|
||||
```
|
||||
const { createReadStream } = require('fs')
|
||||
const { parseSitemapIndex, createSitemap } = require('sitemap')
|
||||
parseSitemapIndex(createReadStream('./example-index.xml')).then(
|
||||
// produces the same xml
|
||||
// you can, of course, more practically modify it or store it
|
||||
(xmlConfig) => console.log(createSitemap(xmlConfig).toString()),
|
||||
(err) => console.log(err)
|
||||
)
|
||||
```
|
||||
@param {Readable} xml what to parse
|
||||
@return {Promise<IndexItem[]>} resolves with list of index items that can be fed into a SitemapIndexStream. Rejects with an Error object.
|
||||
*/
|
||||
export declare function parseSitemapIndex(xml: Readable): Promise<IndexItem[]>;
|
||||
export interface IndexObjectStreamToJSONOptions extends TransformOptions {
|
||||
lineSeparated: boolean;
|
||||
}
|
||||
/**
|
||||
* A Transform that converts a stream of objects into a JSON Array or a line
|
||||
* separated stringified JSON
|
||||
* @param [lineSeparated=false] whether to separate entries by a new line or comma
|
||||
*/
|
||||
export declare class IndexObjectStreamToJSON extends Transform {
|
||||
lineSeparated: boolean;
|
||||
firstWritten: boolean;
|
||||
constructor(opts?: IndexObjectStreamToJSONOptions);
|
||||
_transform(chunk: IndexItem, encoding: string, cb: TransformCallback): void;
|
||||
_flush(cb: TransformCallback): void;
|
||||
}
|
||||
export {};
|
||||
182
node_modules/sitemap/dist/lib/sitemap-index-parser.js
generated
vendored
Normal file
182
node_modules/sitemap/dist/lib/sitemap-index-parser.js
generated
vendored
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
"use strict";
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.IndexObjectStreamToJSON = exports.parseSitemapIndex = exports.XMLToSitemapIndexStream = void 0;
|
||||
const sax_1 = __importDefault(require("sax"));
|
||||
const stream_1 = require("stream");
|
||||
const types_1 = require("./types");
|
||||
function isValidTagName(tagName) {
|
||||
// This only works because the enum name and value are the same
|
||||
return tagName in types_1.IndexTagNames;
|
||||
}
|
||||
function tagTemplate() {
|
||||
return {
|
||||
url: '',
|
||||
};
|
||||
}
|
||||
const defaultLogger = (level, ...message) => console[level](...message);
|
||||
const defaultStreamOpts = {
|
||||
logger: defaultLogger,
|
||||
};
|
||||
// TODO does this need to end with `options`
|
||||
/**
|
||||
* Takes a stream of xml and transforms it into a stream of IndexItems
|
||||
* Use this to parse existing sitemap indices into config options compatible with this library
|
||||
*/
|
||||
class XMLToSitemapIndexStream extends stream_1.Transform {
|
||||
constructor(opts = defaultStreamOpts) {
|
||||
var _a;
|
||||
opts.objectMode = true;
|
||||
super(opts);
|
||||
this.saxStream = sax_1.default.createStream(true, {
|
||||
xmlns: true,
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
strictEntities: true,
|
||||
trim: true,
|
||||
});
|
||||
this.level = opts.level || types_1.ErrorLevel.WARN;
|
||||
if (this.level !== types_1.ErrorLevel.SILENT && opts.logger !== false) {
|
||||
this.logger = (_a = opts.logger) !== null && _a !== void 0 ? _a : defaultLogger;
|
||||
}
|
||||
else {
|
||||
this.logger = () => undefined;
|
||||
}
|
||||
let currentItem = tagTemplate();
|
||||
let currentTag;
|
||||
this.saxStream.on('opentagstart', (tag) => {
|
||||
currentTag = tag.name;
|
||||
});
|
||||
this.saxStream.on('opentag', (tag) => {
|
||||
if (!isValidTagName(tag.name)) {
|
||||
this.logger('warn', 'unhandled tag', tag.name);
|
||||
}
|
||||
});
|
||||
this.saxStream.on('text', (text) => {
|
||||
switch (currentTag) {
|
||||
case types_1.IndexTagNames.loc:
|
||||
currentItem.url = text;
|
||||
break;
|
||||
case types_1.IndexTagNames.lastmod:
|
||||
currentItem.lastmod = text;
|
||||
break;
|
||||
default:
|
||||
this.logger('log', 'unhandled text for tag:', currentTag, `'${text}'`);
|
||||
break;
|
||||
}
|
||||
});
|
||||
this.saxStream.on('cdata', (_text) => {
|
||||
switch (currentTag) {
|
||||
default:
|
||||
this.logger('log', 'unhandled cdata for tag:', currentTag);
|
||||
break;
|
||||
}
|
||||
});
|
||||
this.saxStream.on('attribute', (attr) => {
|
||||
switch (currentTag) {
|
||||
case types_1.IndexTagNames.sitemapindex:
|
||||
break;
|
||||
default:
|
||||
this.logger('log', 'unhandled attr', currentTag, attr.name);
|
||||
}
|
||||
});
|
||||
this.saxStream.on('closetag', (tag) => {
|
||||
switch (tag) {
|
||||
case types_1.IndexTagNames.sitemap:
|
||||
this.push(currentItem);
|
||||
currentItem = tagTemplate();
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
});
|
||||
}
|
||||
_transform(data, encoding, callback) {
|
||||
try {
|
||||
// correcting the type here can be done without making it a breaking change
|
||||
// TODO fix this
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
this.saxStream.write(data, encoding);
|
||||
callback();
|
||||
}
|
||||
catch (error) {
|
||||
callback(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
exports.XMLToSitemapIndexStream = XMLToSitemapIndexStream;
|
||||
/**
|
||||
Read xml and resolve with the configuration that would produce it or reject with
|
||||
an error
|
||||
```
|
||||
const { createReadStream } = require('fs')
|
||||
const { parseSitemapIndex, createSitemap } = require('sitemap')
|
||||
parseSitemapIndex(createReadStream('./example-index.xml')).then(
|
||||
// produces the same xml
|
||||
// you can, of course, more practically modify it or store it
|
||||
(xmlConfig) => console.log(createSitemap(xmlConfig).toString()),
|
||||
(err) => console.log(err)
|
||||
)
|
||||
```
|
||||
@param {Readable} xml what to parse
|
||||
@return {Promise<IndexItem[]>} resolves with list of index items that can be fed into a SitemapIndexStream. Rejects with an Error object.
|
||||
*/
|
||||
async function parseSitemapIndex(xml) {
|
||||
const urls = [];
|
||||
return new Promise((resolve, reject) => {
|
||||
xml
|
||||
.pipe(new XMLToSitemapIndexStream())
|
||||
.on('data', (smi) => urls.push(smi))
|
||||
.on('end', () => {
|
||||
resolve(urls);
|
||||
})
|
||||
.on('error', (error) => {
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
}
|
||||
exports.parseSitemapIndex = parseSitemapIndex;
|
||||
const defaultObjectStreamOpts = {
|
||||
lineSeparated: false,
|
||||
};
|
||||
/**
|
||||
* A Transform that converts a stream of objects into a JSON Array or a line
|
||||
* separated stringified JSON
|
||||
* @param [lineSeparated=false] whether to separate entries by a new line or comma
|
||||
*/
|
||||
class IndexObjectStreamToJSON extends stream_1.Transform {
|
||||
constructor(opts = defaultObjectStreamOpts) {
|
||||
opts.writableObjectMode = true;
|
||||
super(opts);
|
||||
this.lineSeparated = opts.lineSeparated;
|
||||
this.firstWritten = false;
|
||||
}
|
||||
_transform(chunk, encoding, cb) {
|
||||
if (!this.firstWritten) {
|
||||
this.firstWritten = true;
|
||||
if (!this.lineSeparated) {
|
||||
this.push('[');
|
||||
}
|
||||
}
|
||||
else if (this.lineSeparated) {
|
||||
this.push('\n');
|
||||
}
|
||||
else {
|
||||
this.push(',');
|
||||
}
|
||||
if (chunk) {
|
||||
this.push(JSON.stringify(chunk));
|
||||
}
|
||||
cb();
|
||||
}
|
||||
_flush(cb) {
|
||||
if (!this.lineSeparated) {
|
||||
this.push(']');
|
||||
}
|
||||
cb();
|
||||
}
|
||||
}
|
||||
exports.IndexObjectStreamToJSON = IndexObjectStreamToJSON;
|
||||
43
node_modules/sitemap/dist/lib/sitemap-index-stream.d.ts
generated
vendored
Normal file
43
node_modules/sitemap/dist/lib/sitemap-index-stream.d.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
/// <reference types="node" />
|
||||
import { Transform, TransformOptions, TransformCallback } from 'stream';
|
||||
import { IndexItem, SitemapItemLoose, ErrorLevel } from './types';
|
||||
import { SitemapStream } from './sitemap-stream';
|
||||
import { WriteStream } from 'fs';
|
||||
export declare enum IndexTagNames {
|
||||
sitemap = "sitemap",
|
||||
loc = "loc",
|
||||
lastmod = "lastmod"
|
||||
}
|
||||
export interface SitemapIndexStreamOptions extends TransformOptions {
|
||||
lastmodDateOnly?: boolean;
|
||||
level?: ErrorLevel;
|
||||
xslUrl?: string;
|
||||
}
|
||||
export declare class SitemapIndexStream extends Transform {
|
||||
lastmodDateOnly: boolean;
|
||||
level: ErrorLevel;
|
||||
xslUrl?: string;
|
||||
private hasHeadOutput;
|
||||
constructor(opts?: SitemapIndexStreamOptions);
|
||||
_transform(item: IndexItem | string, encoding: string, callback: TransformCallback): void;
|
||||
_flush(cb: TransformCallback): void;
|
||||
}
|
||||
declare type getSitemapStream = (i: number) => [IndexItem | string, SitemapStream, WriteStream];
|
||||
export interface SitemapAndIndexStreamOptions extends SitemapIndexStreamOptions {
|
||||
level?: ErrorLevel;
|
||||
limit?: number;
|
||||
getSitemapStream: getSitemapStream;
|
||||
}
|
||||
export declare class SitemapAndIndexStream extends SitemapIndexStream {
|
||||
private i;
|
||||
private getSitemapStream;
|
||||
private currentSitemap;
|
||||
private currentSitemapPipeline?;
|
||||
private idxItem;
|
||||
private limit;
|
||||
constructor(opts: SitemapAndIndexStreamOptions);
|
||||
_writeSMI(item: SitemapItemLoose): void;
|
||||
_transform(item: SitemapItemLoose, encoding: string, callback: TransformCallback): void;
|
||||
_flush(cb: TransformCallback): void;
|
||||
}
|
||||
export {};
|
||||
103
node_modules/sitemap/dist/lib/sitemap-index-stream.js
generated
vendored
Normal file
103
node_modules/sitemap/dist/lib/sitemap-index-stream.js
generated
vendored
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.SitemapAndIndexStream = exports.SitemapIndexStream = exports.IndexTagNames = void 0;
|
||||
const stream_1 = require("stream");
|
||||
const types_1 = require("./types");
|
||||
const sitemap_stream_1 = require("./sitemap-stream");
|
||||
const sitemap_xml_1 = require("./sitemap-xml");
|
||||
var IndexTagNames;
|
||||
(function (IndexTagNames) {
|
||||
IndexTagNames["sitemap"] = "sitemap";
|
||||
IndexTagNames["loc"] = "loc";
|
||||
IndexTagNames["lastmod"] = "lastmod";
|
||||
})(IndexTagNames = exports.IndexTagNames || (exports.IndexTagNames = {}));
|
||||
const xmlDec = '<?xml version="1.0" encoding="UTF-8"?>';
|
||||
const sitemapIndexTagStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
|
||||
const closetag = '</sitemapindex>';
|
||||
const defaultStreamOpts = {};
|
||||
class SitemapIndexStream extends stream_1.Transform {
|
||||
constructor(opts = defaultStreamOpts) {
|
||||
var _a;
|
||||
opts.objectMode = true;
|
||||
super(opts);
|
||||
this.hasHeadOutput = false;
|
||||
this.lastmodDateOnly = opts.lastmodDateOnly || false;
|
||||
this.level = (_a = opts.level) !== null && _a !== void 0 ? _a : types_1.ErrorLevel.WARN;
|
||||
this.xslUrl = opts.xslUrl;
|
||||
}
|
||||
_transform(item, encoding, callback) {
|
||||
if (!this.hasHeadOutput) {
|
||||
this.hasHeadOutput = true;
|
||||
let stylesheet = '';
|
||||
if (this.xslUrl) {
|
||||
stylesheet = (0, sitemap_stream_1.stylesheetInclude)(this.xslUrl);
|
||||
}
|
||||
this.push(xmlDec + stylesheet + sitemapIndexTagStart);
|
||||
}
|
||||
this.push((0, sitemap_xml_1.otag)(IndexTagNames.sitemap));
|
||||
if (typeof item === 'string') {
|
||||
this.push((0, sitemap_xml_1.element)(IndexTagNames.loc, item));
|
||||
}
|
||||
else {
|
||||
this.push((0, sitemap_xml_1.element)(IndexTagNames.loc, item.url));
|
||||
if (item.lastmod) {
|
||||
const lastmod = new Date(item.lastmod).toISOString();
|
||||
this.push((0, sitemap_xml_1.element)(IndexTagNames.lastmod, this.lastmodDateOnly ? lastmod.slice(0, 10) : lastmod));
|
||||
}
|
||||
}
|
||||
this.push((0, sitemap_xml_1.ctag)(IndexTagNames.sitemap));
|
||||
callback();
|
||||
}
|
||||
_flush(cb) {
|
||||
this.push(closetag);
|
||||
cb();
|
||||
}
|
||||
}
|
||||
exports.SitemapIndexStream = SitemapIndexStream;
|
||||
// const defaultSIStreamOpts: SitemapAndIndexStreamOptions = {};
|
||||
class SitemapAndIndexStream extends SitemapIndexStream {
|
||||
constructor(opts) {
|
||||
var _a;
|
||||
opts.objectMode = true;
|
||||
super(opts);
|
||||
this.i = 0;
|
||||
this.getSitemapStream = opts.getSitemapStream;
|
||||
[this.idxItem, this.currentSitemap, this.currentSitemapPipeline] =
|
||||
this.getSitemapStream(0);
|
||||
this.limit = (_a = opts.limit) !== null && _a !== void 0 ? _a : 45000;
|
||||
}
|
||||
_writeSMI(item) {
|
||||
this.currentSitemap.write(item);
|
||||
this.i++;
|
||||
}
|
||||
_transform(item, encoding, callback) {
|
||||
var _a;
|
||||
if (this.i === 0) {
|
||||
this._writeSMI(item);
|
||||
super._transform(this.idxItem, encoding, callback);
|
||||
}
|
||||
else if (this.i % this.limit === 0) {
|
||||
const onFinish = () => {
|
||||
const [idxItem, currentSitemap, currentSitemapPipeline] = this.getSitemapStream(this.i / this.limit);
|
||||
this.currentSitemap = currentSitemap;
|
||||
this.currentSitemapPipeline = currentSitemapPipeline;
|
||||
this._writeSMI(item);
|
||||
// push to index stream
|
||||
super._transform(idxItem, encoding, callback);
|
||||
};
|
||||
(_a = this.currentSitemapPipeline) === null || _a === void 0 ? void 0 : _a.on('finish', onFinish);
|
||||
this.currentSitemap.end(!this.currentSitemapPipeline ? onFinish : undefined);
|
||||
}
|
||||
else {
|
||||
this._writeSMI(item);
|
||||
callback();
|
||||
}
|
||||
}
|
||||
_flush(cb) {
|
||||
var _a;
|
||||
const onFinish = () => super._flush(cb);
|
||||
(_a = this.currentSitemapPipeline) === null || _a === void 0 ? void 0 : _a.on('finish', onFinish);
|
||||
this.currentSitemap.end(!this.currentSitemapPipeline ? onFinish : undefined);
|
||||
}
|
||||
}
|
||||
exports.SitemapAndIndexStream = SitemapAndIndexStream;
|
||||
25
node_modules/sitemap/dist/lib/sitemap-item-stream.d.ts
generated
vendored
Normal file
25
node_modules/sitemap/dist/lib/sitemap-item-stream.d.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
/// <reference types="node" />
|
||||
import { Transform, TransformOptions, TransformCallback } from 'stream';
|
||||
import { SitemapItem, ErrorLevel } from './types';
|
||||
export interface StringObj {
|
||||
[index: string]: any;
|
||||
}
|
||||
export interface SitemapItemStreamOptions extends TransformOptions {
|
||||
level?: ErrorLevel;
|
||||
}
|
||||
/**
|
||||
* Takes a stream of SitemapItemOptions and spits out xml for each
|
||||
* @example
|
||||
* // writes <url><loc>https://example.com</loc><url><url><loc>https://example.com/2</loc><url>
|
||||
* const smis = new SitemapItemStream({level: 'warn'})
|
||||
* smis.pipe(writestream)
|
||||
* smis.write({url: 'https://example.com', img: [], video: [], links: []})
|
||||
* smis.write({url: 'https://example.com/2', img: [], video: [], links: []})
|
||||
* smis.end()
|
||||
* @param level - Error level
|
||||
*/
|
||||
export declare class SitemapItemStream extends Transform {
|
||||
level: ErrorLevel;
|
||||
constructor(opts?: SitemapItemStreamOptions);
|
||||
_transform(item: SitemapItem, encoding: string, callback: TransformCallback): void;
|
||||
}
|
||||
193
node_modules/sitemap/dist/lib/sitemap-item-stream.js
generated
vendored
Normal file
193
node_modules/sitemap/dist/lib/sitemap-item-stream.js
generated
vendored
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.SitemapItemStream = void 0;
|
||||
const stream_1 = require("stream");
|
||||
const errors_1 = require("./errors");
|
||||
const types_1 = require("./types");
|
||||
const sitemap_xml_1 = require("./sitemap-xml");
|
||||
function attrBuilder(conf, keys) {
|
||||
if (typeof keys === 'string') {
|
||||
keys = [keys];
|
||||
}
|
||||
const iv = {};
|
||||
return keys.reduce((attrs, key) => {
|
||||
// eslint-disable-next-line
|
||||
if (conf[key] !== undefined) {
|
||||
const keyAr = key.split(':');
|
||||
if (keyAr.length !== 2) {
|
||||
throw new errors_1.InvalidAttr(key);
|
||||
}
|
||||
attrs[keyAr[1]] = conf[key];
|
||||
}
|
||||
return attrs;
|
||||
}, iv);
|
||||
}
|
||||
/**
|
||||
* Takes a stream of SitemapItemOptions and spits out xml for each
|
||||
* @example
|
||||
* // writes <url><loc>https://example.com</loc><url><url><loc>https://example.com/2</loc><url>
|
||||
* const smis = new SitemapItemStream({level: 'warn'})
|
||||
* smis.pipe(writestream)
|
||||
* smis.write({url: 'https://example.com', img: [], video: [], links: []})
|
||||
* smis.write({url: 'https://example.com/2', img: [], video: [], links: []})
|
||||
* smis.end()
|
||||
* @param level - Error level
|
||||
*/
|
||||
class SitemapItemStream extends stream_1.Transform {
|
||||
constructor(opts = { level: types_1.ErrorLevel.WARN }) {
|
||||
opts.objectMode = true;
|
||||
super(opts);
|
||||
this.level = opts.level || types_1.ErrorLevel.WARN;
|
||||
}
|
||||
_transform(item, encoding, callback) {
|
||||
this.push((0, sitemap_xml_1.otag)(types_1.TagNames.url));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames.loc, item.url));
|
||||
if (item.lastmod) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames.lastmod, item.lastmod));
|
||||
}
|
||||
if (item.changefreq) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames.changefreq, item.changefreq));
|
||||
}
|
||||
if (item.priority !== undefined && item.priority !== null) {
|
||||
if (item.fullPrecisionPriority) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames.priority, item.priority.toString()));
|
||||
}
|
||||
else {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames.priority, item.priority.toFixed(1)));
|
||||
}
|
||||
}
|
||||
item.video.forEach((video) => {
|
||||
this.push((0, sitemap_xml_1.otag)(types_1.TagNames['video:video']));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:thumbnail_loc'], video.thumbnail_loc));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:title'], video.title));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:description'], video.description));
|
||||
if (video.content_loc) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:content_loc'], video.content_loc));
|
||||
}
|
||||
if (video.player_loc) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:player_loc'], attrBuilder(video, [
|
||||
'player_loc:autoplay',
|
||||
'player_loc:allow_embed',
|
||||
]), video.player_loc));
|
||||
}
|
||||
if (video.duration) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:duration'], video.duration.toString()));
|
||||
}
|
||||
if (video.expiration_date) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:expiration_date'], video.expiration_date));
|
||||
}
|
||||
if (video.rating !== undefined) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:rating'], video.rating.toString()));
|
||||
}
|
||||
if (video.view_count !== undefined) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:view_count'], video.view_count.toString()));
|
||||
}
|
||||
if (video.publication_date) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:publication_date'], video.publication_date));
|
||||
}
|
||||
for (const tag of video.tag) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:tag'], tag));
|
||||
}
|
||||
if (video.category) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:category'], video.category));
|
||||
}
|
||||
if (video.family_friendly) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:family_friendly'], video.family_friendly));
|
||||
}
|
||||
if (video.restriction) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:restriction'], attrBuilder(video, 'restriction:relationship'), video.restriction));
|
||||
}
|
||||
if (video.gallery_loc) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:gallery_loc'], { title: video['gallery_loc:title'] }, video.gallery_loc));
|
||||
}
|
||||
if (video.price) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:price'], attrBuilder(video, [
|
||||
'price:resolution',
|
||||
'price:currency',
|
||||
'price:type',
|
||||
]), video.price));
|
||||
}
|
||||
if (video.requires_subscription) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:requires_subscription'], video.requires_subscription));
|
||||
}
|
||||
if (video.uploader) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:uploader'], attrBuilder(video, 'uploader:info'), video.uploader));
|
||||
}
|
||||
if (video.platform) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:platform'], attrBuilder(video, 'platform:relationship'), video.platform));
|
||||
}
|
||||
if (video.live) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:live'], video.live));
|
||||
}
|
||||
if (video.id) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:id'], { type: 'url' }, video.id));
|
||||
}
|
||||
this.push((0, sitemap_xml_1.ctag)(types_1.TagNames['video:video']));
|
||||
});
|
||||
item.links.forEach((link) => {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['xhtml:link'], {
|
||||
rel: 'alternate',
|
||||
hreflang: link.lang || link.hreflang,
|
||||
href: link.url,
|
||||
}));
|
||||
});
|
||||
if (item.expires) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames.expires, new Date(item.expires).toISOString()));
|
||||
}
|
||||
if (item.androidLink) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['xhtml:link'], {
|
||||
rel: 'alternate',
|
||||
href: item.androidLink,
|
||||
}));
|
||||
}
|
||||
if (item.ampLink) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['xhtml:link'], {
|
||||
rel: 'amphtml',
|
||||
href: item.ampLink,
|
||||
}));
|
||||
}
|
||||
if (item.news) {
|
||||
this.push((0, sitemap_xml_1.otag)(types_1.TagNames['news:news']));
|
||||
this.push((0, sitemap_xml_1.otag)(types_1.TagNames['news:publication']));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:name'], item.news.publication.name));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:language'], item.news.publication.language));
|
||||
this.push((0, sitemap_xml_1.ctag)(types_1.TagNames['news:publication']));
|
||||
if (item.news.access) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:access'], item.news.access));
|
||||
}
|
||||
if (item.news.genres) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:genres'], item.news.genres));
|
||||
}
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:publication_date'], item.news.publication_date));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:title'], item.news.title));
|
||||
if (item.news.keywords) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:keywords'], item.news.keywords));
|
||||
}
|
||||
if (item.news.stock_tickers) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:stock_tickers'], item.news.stock_tickers));
|
||||
}
|
||||
this.push((0, sitemap_xml_1.ctag)(types_1.TagNames['news:news']));
|
||||
}
|
||||
// Image handling
|
||||
item.img.forEach((image) => {
|
||||
this.push((0, sitemap_xml_1.otag)(types_1.TagNames['image:image']));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['image:loc'], image.url));
|
||||
if (image.caption) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['image:caption'], image.caption));
|
||||
}
|
||||
if (image.geoLocation) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['image:geo_location'], image.geoLocation));
|
||||
}
|
||||
if (image.title) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['image:title'], image.title));
|
||||
}
|
||||
if (image.license) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['image:license'], image.license));
|
||||
}
|
||||
this.push((0, sitemap_xml_1.ctag)(types_1.TagNames['image:image']));
|
||||
});
|
||||
this.push((0, sitemap_xml_1.ctag)(types_1.TagNames.url));
|
||||
callback();
|
||||
}
|
||||
}
|
||||
exports.SitemapItemStream = SitemapItemStream;
|
||||
55
node_modules/sitemap/dist/lib/sitemap-parser.d.ts
generated
vendored
Normal file
55
node_modules/sitemap/dist/lib/sitemap-parser.d.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
/// <reference types="node" />
|
||||
import { SAXStream } from 'sax';
|
||||
import { Readable, Transform, TransformOptions, TransformCallback } from 'stream';
|
||||
import { SitemapItem, ErrorLevel } from './types';
|
||||
declare type Logger = (level: 'warn' | 'error' | 'info' | 'log', ...message: Parameters<Console['log']>[0]) => void;
|
||||
export interface XMLToSitemapItemStreamOptions extends TransformOptions {
|
||||
level?: ErrorLevel;
|
||||
logger?: Logger | false;
|
||||
}
|
||||
/**
|
||||
* Takes a stream of xml and transforms it into a stream of SitemapItems
|
||||
* Use this to parse existing sitemaps into config options compatible with this library
|
||||
*/
|
||||
export declare class XMLToSitemapItemStream extends Transform {
|
||||
level: ErrorLevel;
|
||||
logger: Logger;
|
||||
error: Error | null;
|
||||
saxStream: SAXStream;
|
||||
constructor(opts?: XMLToSitemapItemStreamOptions);
|
||||
_transform(data: string, encoding: string, callback: TransformCallback): void;
|
||||
private err;
|
||||
}
|
||||
/**
|
||||
Read xml and resolve with the configuration that would produce it or reject with
|
||||
an error
|
||||
```
|
||||
const { createReadStream } = require('fs')
|
||||
const { parseSitemap, createSitemap } = require('sitemap')
|
||||
parseSitemap(createReadStream('./example.xml')).then(
|
||||
// produces the same xml
|
||||
// you can, of course, more practically modify it or store it
|
||||
(xmlConfig) => console.log(createSitemap(xmlConfig).toString()),
|
||||
(err) => console.log(err)
|
||||
)
|
||||
```
|
||||
@param {Readable} xml what to parse
|
||||
@return {Promise<SitemapItem[]>} resolves with list of sitemap items that can be fed into a SitemapStream. Rejects with an Error object.
|
||||
*/
|
||||
export declare function parseSitemap(xml: Readable): Promise<SitemapItem[]>;
|
||||
export interface ObjectStreamToJSONOptions extends TransformOptions {
|
||||
lineSeparated: boolean;
|
||||
}
|
||||
/**
|
||||
* A Transform that converts a stream of objects into a JSON Array or a line
|
||||
* separated stringified JSON
|
||||
* @param [lineSeparated=false] whether to separate entries by a new line or comma
|
||||
*/
|
||||
export declare class ObjectStreamToJSON extends Transform {
|
||||
lineSeparated: boolean;
|
||||
firstWritten: boolean;
|
||||
constructor(opts?: ObjectStreamToJSONOptions);
|
||||
_transform(chunk: SitemapItem, encoding: string, cb: TransformCallback): void;
|
||||
_flush(cb: TransformCallback): void;
|
||||
}
|
||||
export {};
|
||||
515
node_modules/sitemap/dist/lib/sitemap-parser.js
generated
vendored
Normal file
515
node_modules/sitemap/dist/lib/sitemap-parser.js
generated
vendored
Normal file
|
|
@ -0,0 +1,515 @@
|
|||
"use strict";
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.ObjectStreamToJSON = exports.parseSitemap = exports.XMLToSitemapItemStream = void 0;
|
||||
const sax_1 = __importDefault(require("sax"));
|
||||
const stream_1 = require("stream");
|
||||
const types_1 = require("./types");
|
||||
function isValidTagName(tagName) {
|
||||
// This only works because the enum name and value are the same
|
||||
return tagName in types_1.TagNames;
|
||||
}
|
||||
function tagTemplate() {
|
||||
return {
|
||||
img: [],
|
||||
video: [],
|
||||
links: [],
|
||||
url: '',
|
||||
};
|
||||
}
|
||||
function videoTemplate() {
|
||||
return {
|
||||
tag: [],
|
||||
thumbnail_loc: '',
|
||||
title: '',
|
||||
description: '',
|
||||
};
|
||||
}
|
||||
const imageTemplate = {
|
||||
url: '',
|
||||
};
|
||||
const linkTemplate = {
|
||||
lang: '',
|
||||
url: '',
|
||||
};
|
||||
function newsTemplate() {
|
||||
return {
|
||||
publication: { name: '', language: '' },
|
||||
publication_date: '',
|
||||
title: '',
|
||||
};
|
||||
}
|
||||
const defaultLogger = (level, ...message) => console[level](...message);
|
||||
const defaultStreamOpts = {
|
||||
logger: defaultLogger,
|
||||
};
|
||||
// TODO does this need to end with `options`
|
||||
/**
|
||||
* Takes a stream of xml and transforms it into a stream of SitemapItems
|
||||
* Use this to parse existing sitemaps into config options compatible with this library
|
||||
*/
|
||||
class XMLToSitemapItemStream extends stream_1.Transform {
|
||||
constructor(opts = defaultStreamOpts) {
|
||||
var _a;
|
||||
opts.objectMode = true;
|
||||
super(opts);
|
||||
this.error = null;
|
||||
this.saxStream = sax_1.default.createStream(true, {
|
||||
xmlns: true,
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
strictEntities: true,
|
||||
trim: true,
|
||||
});
|
||||
this.level = opts.level || types_1.ErrorLevel.WARN;
|
||||
if (this.level !== types_1.ErrorLevel.SILENT && opts.logger !== false) {
|
||||
this.logger = (_a = opts.logger) !== null && _a !== void 0 ? _a : defaultLogger;
|
||||
}
|
||||
else {
|
||||
this.logger = () => undefined;
|
||||
}
|
||||
let currentItem = tagTemplate();
|
||||
let currentTag;
|
||||
let currentVideo = videoTemplate();
|
||||
let currentImage = { ...imageTemplate };
|
||||
let currentLink = { ...linkTemplate };
|
||||
let dontpushCurrentLink = false;
|
||||
this.saxStream.on('opentagstart', (tag) => {
|
||||
currentTag = tag.name;
|
||||
if (currentTag.startsWith('news:') && !currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
});
|
||||
this.saxStream.on('opentag', (tag) => {
|
||||
if (isValidTagName(tag.name)) {
|
||||
if (tag.name === 'xhtml:link') {
|
||||
if (typeof tag.attributes.rel === 'string' ||
|
||||
typeof tag.attributes.href === 'string') {
|
||||
return;
|
||||
}
|
||||
if (tag.attributes.rel.value === 'alternate' &&
|
||||
tag.attributes.hreflang) {
|
||||
currentLink.url = tag.attributes.href.value;
|
||||
if (typeof tag.attributes.hreflang === 'string')
|
||||
return;
|
||||
currentLink.lang = tag.attributes.hreflang.value;
|
||||
}
|
||||
else if (tag.attributes.rel.value === 'alternate') {
|
||||
dontpushCurrentLink = true;
|
||||
currentItem.androidLink = tag.attributes.href.value;
|
||||
}
|
||||
else if (tag.attributes.rel.value === 'amphtml') {
|
||||
dontpushCurrentLink = true;
|
||||
currentItem.ampLink = tag.attributes.href.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr for xhtml:link', tag.attributes);
|
||||
this.err(`unhandled attr for xhtml:link ${tag.attributes}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
this.logger('warn', 'unhandled tag', tag.name);
|
||||
this.err(`unhandled tag: ${tag.name}`);
|
||||
}
|
||||
});
|
||||
this.saxStream.on('text', (text) => {
|
||||
switch (currentTag) {
|
||||
case 'mobile:mobile':
|
||||
break;
|
||||
case types_1.TagNames.loc:
|
||||
currentItem.url = text;
|
||||
break;
|
||||
case types_1.TagNames.changefreq:
|
||||
if ((0, types_1.isValidChangeFreq)(text)) {
|
||||
currentItem.changefreq = text;
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames.priority:
|
||||
currentItem.priority = parseFloat(text);
|
||||
break;
|
||||
case types_1.TagNames.lastmod:
|
||||
currentItem.lastmod = text;
|
||||
break;
|
||||
case types_1.TagNames['video:thumbnail_loc']:
|
||||
currentVideo.thumbnail_loc = text;
|
||||
break;
|
||||
case types_1.TagNames['video:tag']:
|
||||
currentVideo.tag.push(text);
|
||||
break;
|
||||
case types_1.TagNames['video:duration']:
|
||||
currentVideo.duration = parseInt(text, 10);
|
||||
break;
|
||||
case types_1.TagNames['video:player_loc']:
|
||||
currentVideo.player_loc = text;
|
||||
break;
|
||||
case types_1.TagNames['video:content_loc']:
|
||||
currentVideo.content_loc = text;
|
||||
break;
|
||||
case types_1.TagNames['video:requires_subscription']:
|
||||
if ((0, types_1.isValidYesNo)(text)) {
|
||||
currentVideo.requires_subscription = text;
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:publication_date']:
|
||||
currentVideo.publication_date = text;
|
||||
break;
|
||||
case types_1.TagNames['video:id']:
|
||||
currentVideo.id = text;
|
||||
break;
|
||||
case types_1.TagNames['video:restriction']:
|
||||
currentVideo.restriction = text;
|
||||
break;
|
||||
case types_1.TagNames['video:view_count']:
|
||||
currentVideo.view_count = parseInt(text, 10);
|
||||
break;
|
||||
case types_1.TagNames['video:uploader']:
|
||||
currentVideo.uploader = text;
|
||||
break;
|
||||
case types_1.TagNames['video:family_friendly']:
|
||||
if ((0, types_1.isValidYesNo)(text)) {
|
||||
currentVideo.family_friendly = text;
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:expiration_date']:
|
||||
currentVideo.expiration_date = text;
|
||||
break;
|
||||
case types_1.TagNames['video:platform']:
|
||||
currentVideo.platform = text;
|
||||
break;
|
||||
case types_1.TagNames['video:price']:
|
||||
currentVideo.price = text;
|
||||
break;
|
||||
case types_1.TagNames['video:rating']:
|
||||
currentVideo.rating = parseFloat(text);
|
||||
break;
|
||||
case types_1.TagNames['video:category']:
|
||||
currentVideo.category = text;
|
||||
break;
|
||||
case types_1.TagNames['video:live']:
|
||||
if ((0, types_1.isValidYesNo)(text)) {
|
||||
currentVideo.live = text;
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:gallery_loc']:
|
||||
currentVideo.gallery_loc = text;
|
||||
break;
|
||||
case types_1.TagNames['image:loc']:
|
||||
currentImage.url = text;
|
||||
break;
|
||||
case types_1.TagNames['image:geo_location']:
|
||||
currentImage.geoLocation = text;
|
||||
break;
|
||||
case types_1.TagNames['image:license']:
|
||||
currentImage.license = text;
|
||||
break;
|
||||
case types_1.TagNames['news:access']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.access = text;
|
||||
break;
|
||||
case types_1.TagNames['news:genres']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.genres = text;
|
||||
break;
|
||||
case types_1.TagNames['news:publication_date']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.publication_date = text;
|
||||
break;
|
||||
case types_1.TagNames['news:keywords']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.keywords = text;
|
||||
break;
|
||||
case types_1.TagNames['news:stock_tickers']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.stock_tickers = text;
|
||||
break;
|
||||
case types_1.TagNames['news:language']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.publication.language = text;
|
||||
break;
|
||||
case types_1.TagNames['video:title']:
|
||||
currentVideo.title += text;
|
||||
break;
|
||||
case types_1.TagNames['video:description']:
|
||||
currentVideo.description += text;
|
||||
break;
|
||||
case types_1.TagNames['news:name']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.publication.name += text;
|
||||
break;
|
||||
case types_1.TagNames['news:title']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.title += text;
|
||||
break;
|
||||
case types_1.TagNames['image:caption']:
|
||||
if (!currentImage.caption) {
|
||||
currentImage.caption = text;
|
||||
}
|
||||
else {
|
||||
currentImage.caption += text;
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['image:title']:
|
||||
if (!currentImage.title) {
|
||||
currentImage.title = text;
|
||||
}
|
||||
else {
|
||||
currentImage.title += text;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
this.logger('log', 'unhandled text for tag:', currentTag, `'${text}'`);
|
||||
this.err(`unhandled text for tag: ${currentTag} '${text}'`);
|
||||
break;
|
||||
}
|
||||
});
|
||||
this.saxStream.on('cdata', (text) => {
|
||||
switch (currentTag) {
|
||||
case types_1.TagNames['video:title']:
|
||||
currentVideo.title += text;
|
||||
break;
|
||||
case types_1.TagNames['video:description']:
|
||||
currentVideo.description += text;
|
||||
break;
|
||||
case types_1.TagNames['news:name']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.publication.name += text;
|
||||
break;
|
||||
case types_1.TagNames['news:title']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.title += text;
|
||||
break;
|
||||
case types_1.TagNames['image:caption']:
|
||||
if (!currentImage.caption) {
|
||||
currentImage.caption = text;
|
||||
}
|
||||
else {
|
||||
currentImage.caption += text;
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['image:title']:
|
||||
if (!currentImage.title) {
|
||||
currentImage.title = text;
|
||||
}
|
||||
else {
|
||||
currentImage.title += text;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
this.logger('log', 'unhandled cdata for tag:', currentTag);
|
||||
this.err(`unhandled cdata for tag: ${currentTag}`);
|
||||
break;
|
||||
}
|
||||
});
|
||||
this.saxStream.on('attribute', (attr) => {
|
||||
switch (currentTag) {
|
||||
case types_1.TagNames['urlset']:
|
||||
case types_1.TagNames['xhtml:link']:
|
||||
case types_1.TagNames['video:id']:
|
||||
break;
|
||||
case types_1.TagNames['video:restriction']:
|
||||
if (attr.name === 'relationship' && (0, types_1.isAllowDeny)(attr.value)) {
|
||||
currentVideo['restriction:relationship'] = attr.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr', currentTag, attr.name);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name}`);
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:price']:
|
||||
if (attr.name === 'type' && (0, types_1.isPriceType)(attr.value)) {
|
||||
currentVideo['price:type'] = attr.value;
|
||||
}
|
||||
else if (attr.name === 'currency') {
|
||||
currentVideo['price:currency'] = attr.value;
|
||||
}
|
||||
else if (attr.name === 'resolution' && (0, types_1.isResolution)(attr.value)) {
|
||||
currentVideo['price:resolution'] = attr.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr for video:price', attr.name);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name}`);
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:player_loc']:
|
||||
if (attr.name === 'autoplay') {
|
||||
currentVideo['player_loc:autoplay'] = attr.value;
|
||||
}
|
||||
else if (attr.name === 'allow_embed' && (0, types_1.isValidYesNo)(attr.value)) {
|
||||
currentVideo['player_loc:allow_embed'] = attr.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr for video:player_loc', attr.name);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name}`);
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:platform']:
|
||||
if (attr.name === 'relationship' && (0, types_1.isAllowDeny)(attr.value)) {
|
||||
currentVideo['platform:relationship'] = attr.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr for video:platform', attr.name, attr.value);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name} ${attr.value}`);
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:gallery_loc']:
|
||||
if (attr.name === 'title') {
|
||||
currentVideo['gallery_loc:title'] = attr.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr for video:galler_loc', attr.name);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name}`);
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:uploader']:
|
||||
if (attr.name === 'info') {
|
||||
currentVideo['uploader:info'] = attr.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr for video:uploader', attr.name);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name}`);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
this.logger('log', 'unhandled attr', currentTag, attr.name);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name}`);
|
||||
}
|
||||
});
|
||||
this.saxStream.on('closetag', (tag) => {
|
||||
switch (tag) {
|
||||
case types_1.TagNames.url:
|
||||
this.push(currentItem);
|
||||
currentItem = tagTemplate();
|
||||
break;
|
||||
case types_1.TagNames['video:video']:
|
||||
currentItem.video.push(currentVideo);
|
||||
currentVideo = videoTemplate();
|
||||
break;
|
||||
case types_1.TagNames['image:image']:
|
||||
currentItem.img.push(currentImage);
|
||||
currentImage = { ...imageTemplate };
|
||||
break;
|
||||
case types_1.TagNames['xhtml:link']:
|
||||
if (!dontpushCurrentLink) {
|
||||
currentItem.links.push(currentLink);
|
||||
}
|
||||
currentLink = { ...linkTemplate };
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
});
|
||||
}
|
||||
_transform(data, encoding, callback) {
|
||||
try {
|
||||
// correcting the type here can be done without making it a breaking change
|
||||
// TODO fix this
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
this.saxStream.write(data, encoding);
|
||||
callback(this.level === types_1.ErrorLevel.THROW ? this.error : null);
|
||||
}
|
||||
catch (error) {
|
||||
callback(error);
|
||||
}
|
||||
}
|
||||
err(msg) {
|
||||
if (!this.error)
|
||||
this.error = new Error(msg);
|
||||
}
|
||||
}
|
||||
exports.XMLToSitemapItemStream = XMLToSitemapItemStream;
|
||||
/**
|
||||
Read xml and resolve with the configuration that would produce it or reject with
|
||||
an error
|
||||
```
|
||||
const { createReadStream } = require('fs')
|
||||
const { parseSitemap, createSitemap } = require('sitemap')
|
||||
parseSitemap(createReadStream('./example.xml')).then(
|
||||
// produces the same xml
|
||||
// you can, of course, more practically modify it or store it
|
||||
(xmlConfig) => console.log(createSitemap(xmlConfig).toString()),
|
||||
(err) => console.log(err)
|
||||
)
|
||||
```
|
||||
@param {Readable} xml what to parse
|
||||
@return {Promise<SitemapItem[]>} resolves with list of sitemap items that can be fed into a SitemapStream. Rejects with an Error object.
|
||||
*/
|
||||
async function parseSitemap(xml) {
|
||||
const urls = [];
|
||||
return new Promise((resolve, reject) => {
|
||||
xml
|
||||
.pipe(new XMLToSitemapItemStream())
|
||||
.on('data', (smi) => urls.push(smi))
|
||||
.on('end', () => {
|
||||
resolve(urls);
|
||||
})
|
||||
.on('error', (error) => {
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
}
|
||||
exports.parseSitemap = parseSitemap;
|
||||
const defaultObjectStreamOpts = {
|
||||
lineSeparated: false,
|
||||
};
|
||||
/**
|
||||
* A Transform that converts a stream of objects into a JSON Array or a line
|
||||
* separated stringified JSON
|
||||
* @param [lineSeparated=false] whether to separate entries by a new line or comma
|
||||
*/
|
||||
class ObjectStreamToJSON extends stream_1.Transform {
|
||||
constructor(opts = defaultObjectStreamOpts) {
|
||||
opts.writableObjectMode = true;
|
||||
super(opts);
|
||||
this.lineSeparated = opts.lineSeparated;
|
||||
this.firstWritten = false;
|
||||
}
|
||||
_transform(chunk, encoding, cb) {
|
||||
if (!this.firstWritten) {
|
||||
this.firstWritten = true;
|
||||
if (!this.lineSeparated) {
|
||||
this.push('[');
|
||||
}
|
||||
}
|
||||
else if (this.lineSeparated) {
|
||||
this.push('\n');
|
||||
}
|
||||
else {
|
||||
this.push(',');
|
||||
}
|
||||
if (chunk) {
|
||||
this.push(JSON.stringify(chunk));
|
||||
}
|
||||
cb();
|
||||
}
|
||||
_flush(cb) {
|
||||
if (!this.lineSeparated) {
|
||||
this.push(']');
|
||||
}
|
||||
cb();
|
||||
}
|
||||
}
|
||||
exports.ObjectStreamToJSON = ObjectStreamToJSON;
|
||||
24
node_modules/sitemap/dist/lib/sitemap-simple.d.ts
generated
vendored
Normal file
24
node_modules/sitemap/dist/lib/sitemap-simple.d.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import { Readable } from 'stream';
|
||||
import { SitemapItemLoose } from './types';
|
||||
/**
|
||||
*
|
||||
* @param {object} options -
|
||||
* @param {string} options.hostname - The hostname for all URLs
|
||||
* @param {string} [options.sitemapHostname] - The hostname for the sitemaps if different than hostname
|
||||
* @param {SitemapItemLoose[] | string | Readable | string[]} options.sourceData - The urls you want to make a sitemap out of.
|
||||
* @param {string} options.destinationDir - where to write the sitemaps and index
|
||||
* @param {string} [options.publicBasePath] - where the sitemaps are relative to the hostname. Defaults to root.
|
||||
* @param {number} [options.limit] - how many URLs to write before switching to a new file. Defaults to 50k
|
||||
* @param {boolean} [options.gzip] - whether to compress the written files. Defaults to true
|
||||
* @returns {Promise<void>} an empty promise that resolves when everything is done
|
||||
*/
|
||||
export declare const simpleSitemapAndIndex: ({ hostname, sitemapHostname, sourceData, destinationDir, limit, gzip, publicBasePath, }: {
|
||||
hostname: string;
|
||||
sitemapHostname?: string | undefined;
|
||||
sourceData: SitemapItemLoose[] | string | Readable | string[];
|
||||
destinationDir: string;
|
||||
publicBasePath?: string | undefined;
|
||||
limit?: number | undefined;
|
||||
gzip?: boolean | undefined;
|
||||
}) => Promise<void>;
|
||||
export default simpleSitemapAndIndex;
|
||||
80
node_modules/sitemap/dist/lib/sitemap-simple.js
generated
vendored
Normal file
80
node_modules/sitemap/dist/lib/sitemap-simple.js
generated
vendored
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.simpleSitemapAndIndex = void 0;
|
||||
const index_1 = require("../index");
|
||||
const zlib_1 = require("zlib");
|
||||
const fs_1 = require("fs");
|
||||
const path_1 = require("path");
|
||||
const stream_1 = require("stream");
|
||||
const util_1 = require("util");
|
||||
const url_1 = require("url");
|
||||
const pipeline = (0, util_1.promisify)(stream_1.pipeline);
|
||||
/**
|
||||
*
|
||||
* @param {object} options -
|
||||
* @param {string} options.hostname - The hostname for all URLs
|
||||
* @param {string} [options.sitemapHostname] - The hostname for the sitemaps if different than hostname
|
||||
* @param {SitemapItemLoose[] | string | Readable | string[]} options.sourceData - The urls you want to make a sitemap out of.
|
||||
* @param {string} options.destinationDir - where to write the sitemaps and index
|
||||
* @param {string} [options.publicBasePath] - where the sitemaps are relative to the hostname. Defaults to root.
|
||||
* @param {number} [options.limit] - how many URLs to write before switching to a new file. Defaults to 50k
|
||||
* @param {boolean} [options.gzip] - whether to compress the written files. Defaults to true
|
||||
* @returns {Promise<void>} an empty promise that resolves when everything is done
|
||||
*/
|
||||
const simpleSitemapAndIndex = async ({ hostname, sitemapHostname = hostname, // if different
|
||||
/**
|
||||
* Pass a line separated list of sitemap items or a stream or an array
|
||||
*/
|
||||
sourceData, destinationDir, limit = 50000, gzip = true, publicBasePath = './', }) => {
|
||||
await fs_1.promises.mkdir(destinationDir, { recursive: true });
|
||||
const sitemapAndIndexStream = new index_1.SitemapAndIndexStream({
|
||||
limit,
|
||||
getSitemapStream: (i) => {
|
||||
const sitemapStream = new index_1.SitemapStream({
|
||||
hostname,
|
||||
});
|
||||
const path = `./sitemap-${i}.xml`;
|
||||
const writePath = (0, path_1.resolve)(destinationDir, path + (gzip ? '.gz' : ''));
|
||||
if (!publicBasePath.endsWith('/')) {
|
||||
publicBasePath += '/';
|
||||
}
|
||||
const publicPath = (0, path_1.normalize)(publicBasePath + path);
|
||||
let pipeline;
|
||||
if (gzip) {
|
||||
pipeline = sitemapStream
|
||||
.pipe((0, zlib_1.createGzip)()) // compress the output of the sitemap
|
||||
.pipe((0, fs_1.createWriteStream)(writePath)); // write it to sitemap-NUMBER.xml
|
||||
}
|
||||
else {
|
||||
pipeline = sitemapStream.pipe((0, fs_1.createWriteStream)(writePath)); // write it to sitemap-NUMBER.xml
|
||||
}
|
||||
return [
|
||||
new url_1.URL(`${publicPath}${gzip ? '.gz' : ''}`, sitemapHostname).toString(),
|
||||
sitemapStream,
|
||||
pipeline,
|
||||
];
|
||||
},
|
||||
});
|
||||
let src;
|
||||
if (typeof sourceData === 'string') {
|
||||
src = (0, index_1.lineSeparatedURLsToSitemapOptions)((0, fs_1.createReadStream)(sourceData));
|
||||
}
|
||||
else if (sourceData instanceof stream_1.Readable) {
|
||||
src = sourceData;
|
||||
}
|
||||
else if (Array.isArray(sourceData)) {
|
||||
src = stream_1.Readable.from(sourceData);
|
||||
}
|
||||
else {
|
||||
throw new Error("unhandled source type. You've passed in data that is not supported");
|
||||
}
|
||||
const writePath = (0, path_1.resolve)(destinationDir, `./sitemap-index.xml${gzip ? '.gz' : ''}`);
|
||||
if (gzip) {
|
||||
return pipeline(src, sitemapAndIndexStream, (0, zlib_1.createGzip)(), (0, fs_1.createWriteStream)(writePath));
|
||||
}
|
||||
else {
|
||||
return pipeline(src, sitemapAndIndexStream, (0, fs_1.createWriteStream)(writePath));
|
||||
}
|
||||
};
|
||||
exports.simpleSitemapAndIndex = simpleSitemapAndIndex;
|
||||
exports.default = exports.simpleSitemapAndIndex;
|
||||
45
node_modules/sitemap/dist/lib/sitemap-stream.d.ts
generated
vendored
Normal file
45
node_modules/sitemap/dist/lib/sitemap-stream.d.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
/// <reference types="node" />
|
||||
import { Transform, TransformOptions, TransformCallback, Readable } from 'stream';
|
||||
import { SitemapItemLoose, ErrorLevel, ErrorHandler } from './types';
|
||||
export declare const stylesheetInclude: (url: string) => string;
|
||||
export interface NSArgs {
|
||||
news: boolean;
|
||||
video: boolean;
|
||||
xhtml: boolean;
|
||||
image: boolean;
|
||||
custom?: string[];
|
||||
}
|
||||
export declare const closetag = "</urlset>";
|
||||
export interface SitemapStreamOptions extends TransformOptions {
|
||||
hostname?: string;
|
||||
level?: ErrorLevel;
|
||||
lastmodDateOnly?: boolean;
|
||||
xmlns?: NSArgs;
|
||||
xslUrl?: string;
|
||||
errorHandler?: ErrorHandler;
|
||||
}
|
||||
/**
|
||||
* A [Transform](https://nodejs.org/api/stream.html#stream_implementing_a_transform_stream)
|
||||
* for turning a
|
||||
* [Readable stream](https://nodejs.org/api/stream.html#stream_readable_streams)
|
||||
* of either [SitemapItemOptions](#sitemap-item-options) or url strings into a
|
||||
* Sitemap. The readable stream it transforms **must** be in object mode.
|
||||
*/
|
||||
export declare class SitemapStream extends Transform {
|
||||
hostname?: string;
|
||||
level: ErrorLevel;
|
||||
hasHeadOutput: boolean;
|
||||
xmlNS: NSArgs;
|
||||
xslUrl?: string;
|
||||
errorHandler?: ErrorHandler;
|
||||
private smiStream;
|
||||
lastmodDateOnly: boolean;
|
||||
constructor(opts?: SitemapStreamOptions);
|
||||
_transform(item: SitemapItemLoose, encoding: string, callback: TransformCallback): void;
|
||||
_flush(cb: TransformCallback): void;
|
||||
}
|
||||
/**
|
||||
* Takes a stream returns a promise that resolves when stream emits finish
|
||||
* @param stream what you want wrapped in a promise
|
||||
*/
|
||||
export declare function streamToPromise(stream: Readable): Promise<Buffer>;
|
||||
113
node_modules/sitemap/dist/lib/sitemap-stream.js
generated
vendored
Normal file
113
node_modules/sitemap/dist/lib/sitemap-stream.js
generated
vendored
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.streamToPromise = exports.SitemapStream = exports.closetag = exports.stylesheetInclude = void 0;
|
||||
const stream_1 = require("stream");
|
||||
const types_1 = require("./types");
|
||||
const utils_1 = require("./utils");
|
||||
const sitemap_item_stream_1 = require("./sitemap-item-stream");
|
||||
const errors_1 = require("./errors");
|
||||
const xmlDec = '<?xml version="1.0" encoding="UTF-8"?>';
|
||||
const stylesheetInclude = (url) => {
|
||||
return `<?xml-stylesheet type="text/xsl" href="${url}"?>`;
|
||||
};
|
||||
exports.stylesheetInclude = stylesheetInclude;
|
||||
const urlsetTagStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"';
|
||||
const getURLSetNs = ({ news, video, image, xhtml, custom }, xslURL) => {
|
||||
let ns = xmlDec;
|
||||
if (xslURL) {
|
||||
ns += (0, exports.stylesheetInclude)(xslURL);
|
||||
}
|
||||
ns += urlsetTagStart;
|
||||
if (news) {
|
||||
ns += ' xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"';
|
||||
}
|
||||
if (xhtml) {
|
||||
ns += ' xmlns:xhtml="http://www.w3.org/1999/xhtml"';
|
||||
}
|
||||
if (image) {
|
||||
ns += ' xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"';
|
||||
}
|
||||
if (video) {
|
||||
ns += ' xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"';
|
||||
}
|
||||
if (custom) {
|
||||
ns += ' ' + custom.join(' ');
|
||||
}
|
||||
return ns + '>';
|
||||
};
|
||||
exports.closetag = '</urlset>';
|
||||
const defaultXMLNS = {
|
||||
news: true,
|
||||
xhtml: true,
|
||||
image: true,
|
||||
video: true,
|
||||
};
|
||||
const defaultStreamOpts = {
|
||||
xmlns: defaultXMLNS,
|
||||
};
|
||||
/**
|
||||
* A [Transform](https://nodejs.org/api/stream.html#stream_implementing_a_transform_stream)
|
||||
* for turning a
|
||||
* [Readable stream](https://nodejs.org/api/stream.html#stream_readable_streams)
|
||||
* of either [SitemapItemOptions](#sitemap-item-options) or url strings into a
|
||||
* Sitemap. The readable stream it transforms **must** be in object mode.
|
||||
*/
|
||||
class SitemapStream extends stream_1.Transform {
|
||||
constructor(opts = defaultStreamOpts) {
|
||||
opts.objectMode = true;
|
||||
super(opts);
|
||||
this.hasHeadOutput = false;
|
||||
this.hostname = opts.hostname;
|
||||
this.level = opts.level || types_1.ErrorLevel.WARN;
|
||||
this.errorHandler = opts.errorHandler;
|
||||
this.smiStream = new sitemap_item_stream_1.SitemapItemStream({ level: opts.level });
|
||||
this.smiStream.on('data', (data) => this.push(data));
|
||||
this.lastmodDateOnly = opts.lastmodDateOnly || false;
|
||||
this.xmlNS = opts.xmlns || defaultXMLNS;
|
||||
this.xslUrl = opts.xslUrl;
|
||||
}
|
||||
_transform(item, encoding, callback) {
|
||||
if (!this.hasHeadOutput) {
|
||||
this.hasHeadOutput = true;
|
||||
this.push(getURLSetNs(this.xmlNS, this.xslUrl));
|
||||
}
|
||||
this.smiStream.write((0, utils_1.validateSMIOptions)((0, utils_1.normalizeURL)(item, this.hostname, this.lastmodDateOnly), this.level, this.errorHandler));
|
||||
callback();
|
||||
}
|
||||
_flush(cb) {
|
||||
if (!this.hasHeadOutput) {
|
||||
cb(new errors_1.EmptySitemap());
|
||||
}
|
||||
else {
|
||||
this.push(exports.closetag);
|
||||
cb();
|
||||
}
|
||||
}
|
||||
}
|
||||
exports.SitemapStream = SitemapStream;
|
||||
/**
|
||||
* Takes a stream returns a promise that resolves when stream emits finish
|
||||
* @param stream what you want wrapped in a promise
|
||||
*/
|
||||
function streamToPromise(stream) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const drain = [];
|
||||
stream
|
||||
.pipe(new stream_1.Writable({
|
||||
write(chunk, enc, next) {
|
||||
drain.push(chunk);
|
||||
next();
|
||||
},
|
||||
}))
|
||||
.on('error', reject)
|
||||
.on('finish', () => {
|
||||
if (!drain.length) {
|
||||
reject(new errors_1.EmptyStream());
|
||||
}
|
||||
else {
|
||||
resolve(Buffer.concat(drain));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
exports.streamToPromise = streamToPromise;
|
||||
9
node_modules/sitemap/dist/lib/sitemap-xml.d.ts
generated
vendored
Normal file
9
node_modules/sitemap/dist/lib/sitemap-xml.d.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
import { TagNames } from './types';
|
||||
import { StringObj } from './sitemap-item-stream';
|
||||
import { IndexTagNames } from './sitemap-index-stream';
|
||||
export declare function text(txt: string): string;
|
||||
export declare function otag(nodeName: TagNames | IndexTagNames, attrs?: StringObj, selfClose?: boolean): string;
|
||||
export declare function ctag(nodeName: TagNames | IndexTagNames): string;
|
||||
export declare function element(nodeName: TagNames, attrs: StringObj, innerText: string): string;
|
||||
export declare function element(nodeName: TagNames | IndexTagNames, innerText: string): string;
|
||||
export declare function element(nodeName: TagNames, attrs: StringObj): string;
|
||||
47
node_modules/sitemap/dist/lib/sitemap-xml.js
generated
vendored
Normal file
47
node_modules/sitemap/dist/lib/sitemap-xml.js
generated
vendored
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.element = exports.ctag = exports.otag = exports.text = void 0;
|
||||
const invalidXMLUnicodeRegex =
|
||||
// eslint-disable-next-line no-control-regex
|
||||
/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u0084\u0086-\u009F\uD800-\uDFFF\uFDD0-\uFDDF\u{1FFFE}-\u{1FFFF}\u{2FFFE}-\u{2FFFF}\u{3FFFE}-\u{3FFFF}\u{4FFFE}-\u{4FFFF}\u{5FFFE}-\u{5FFFF}\u{6FFFE}-\u{6FFFF}\u{7FFFE}-\u{7FFFF}\u{8FFFE}-\u{8FFFF}\u{9FFFE}-\u{9FFFF}\u{AFFFE}-\u{AFFFF}\u{BFFFE}-\u{BFFFF}\u{CFFFE}-\u{CFFFF}\u{DFFFE}-\u{DFFFF}\u{EFFFE}-\u{EFFFF}\u{FFFFE}-\u{FFFFF}\u{10FFFE}-\u{10FFFF}]/gu;
|
||||
const amp = /&/g;
|
||||
const lt = /</g;
|
||||
const apos = /'/g;
|
||||
const quot = /"/g;
|
||||
function text(txt) {
|
||||
return txt
|
||||
.replace(amp, '&')
|
||||
.replace(lt, '<')
|
||||
.replace(invalidXMLUnicodeRegex, '');
|
||||
}
|
||||
exports.text = text;
|
||||
function otag(nodeName, attrs, selfClose = false) {
|
||||
let attrstr = '';
|
||||
for (const k in attrs) {
|
||||
const val = attrs[k]
|
||||
.replace(amp, '&')
|
||||
.replace(lt, '<')
|
||||
.replace(apos, ''')
|
||||
.replace(quot, '"')
|
||||
.replace(invalidXMLUnicodeRegex, '');
|
||||
attrstr += ` ${k}="${val}"`;
|
||||
}
|
||||
return `<${nodeName}${attrstr}${selfClose ? '/' : ''}>`;
|
||||
}
|
||||
exports.otag = otag;
|
||||
function ctag(nodeName) {
|
||||
return `</${nodeName}>`;
|
||||
}
|
||||
exports.ctag = ctag;
|
||||
function element(nodeName, attrs, innerText) {
|
||||
if (typeof attrs === 'string') {
|
||||
return otag(nodeName) + text(attrs) + ctag(nodeName);
|
||||
}
|
||||
else if (innerText) {
|
||||
return otag(nodeName, attrs) + text(innerText) + ctag(nodeName);
|
||||
}
|
||||
else {
|
||||
return otag(nodeName, attrs, true);
|
||||
}
|
||||
}
|
||||
exports.element = element;
|
||||
393
node_modules/sitemap/dist/lib/types.d.ts
generated
vendored
Normal file
393
node_modules/sitemap/dist/lib/types.d.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,393 @@
|
|||
/// <reference types="node" />
|
||||
import { URL } from 'url';
|
||||
/**
|
||||
* How frequently the page is likely to change. This value provides general
|
||||
* information to search engines and may not correlate exactly to how often they crawl the page. Please note that the
|
||||
* value of this tag is considered a hint and not a command. See
|
||||
* <https://www.sitemaps.org/protocol.html#xmlTagDefinitions> for the acceptable
|
||||
* values
|
||||
*/
|
||||
export declare enum EnumChangefreq {
|
||||
DAILY = "daily",
|
||||
MONTHLY = "monthly",
|
||||
ALWAYS = "always",
|
||||
HOURLY = "hourly",
|
||||
WEEKLY = "weekly",
|
||||
YEARLY = "yearly",
|
||||
NEVER = "never"
|
||||
}
|
||||
export declare const validators: {
|
||||
[index: string]: RegExp;
|
||||
};
|
||||
export declare function isPriceType(pt: string | PriceType): pt is PriceType;
|
||||
export declare function isResolution(res: string): res is Resolution;
|
||||
export declare const CHANGEFREQ: EnumChangefreq[];
|
||||
export declare function isValidChangeFreq(freq: string): freq is EnumChangefreq;
|
||||
export declare enum EnumYesNo {
|
||||
YES = "YES",
|
||||
NO = "NO",
|
||||
Yes = "Yes",
|
||||
No = "No",
|
||||
yes = "yes",
|
||||
no = "no"
|
||||
}
|
||||
export declare function isValidYesNo(yn: string): yn is EnumYesNo;
|
||||
export declare enum EnumAllowDeny {
|
||||
ALLOW = "allow",
|
||||
DENY = "deny"
|
||||
}
|
||||
export declare function isAllowDeny(ad: string): ad is EnumAllowDeny;
|
||||
/**
|
||||
* https://support.google.com/webmasters/answer/74288?hl=en&ref_topic=4581190
|
||||
*/
|
||||
export interface NewsItem {
|
||||
access?: 'Registration' | 'Subscription';
|
||||
publication: {
|
||||
name: string;
|
||||
/**
|
||||
* The `<language>` is the language of your publication. Use an ISO 639
|
||||
* language code (2 or 3 letters).
|
||||
*/
|
||||
language: string;
|
||||
};
|
||||
/**
|
||||
* @example 'PressRelease, Blog'
|
||||
*/
|
||||
genres?: string;
|
||||
/**
|
||||
* Article publication date in W3C format, using either the "complete date" (YYYY-MM-DD) format or the "complete date
|
||||
* plus hours, minutes, and seconds"
|
||||
*/
|
||||
publication_date: string;
|
||||
/**
|
||||
* The title of the news article
|
||||
* @example 'Companies A, B in Merger Talks'
|
||||
*/
|
||||
title: string;
|
||||
/**
|
||||
* @example 'business, merger, acquisition'
|
||||
*/
|
||||
keywords?: string;
|
||||
/**
|
||||
* @example 'NASDAQ:A, NASDAQ:B'
|
||||
*/
|
||||
stock_tickers?: string;
|
||||
}
|
||||
/**
|
||||
* Sitemap Image
|
||||
* https://support.google.com/webmasters/answer/178636?hl=en&ref_topic=4581190
|
||||
*/
|
||||
export interface Img {
|
||||
/**
|
||||
* The URL of the image
|
||||
* @example 'https://example.com/image.jpg'
|
||||
*/
|
||||
url: string;
|
||||
/**
|
||||
* The caption of the image
|
||||
* @example 'Thanksgiving dinner'
|
||||
*/
|
||||
caption?: string;
|
||||
/**
|
||||
* The title of the image
|
||||
* @example 'Star Wars EP IV'
|
||||
*/
|
||||
title?: string;
|
||||
/**
|
||||
* The geographic location of the image.
|
||||
* @example 'Limerick, Ireland'
|
||||
*/
|
||||
geoLocation?: string;
|
||||
/**
|
||||
* A URL to the license of the image.
|
||||
* @example 'https://example.com/license.txt'
|
||||
*/
|
||||
license?: string;
|
||||
}
|
||||
interface VideoItemBase {
|
||||
/**
|
||||
* A URL pointing to the video thumbnail image file
|
||||
* @example "https://rtv3-img-roosterteeth.akamaized.net/store/0e841100-289b-4184-ae30-b6a16736960a.jpg/sm/thumb3.jpg"
|
||||
*/
|
||||
thumbnail_loc: string;
|
||||
/**
|
||||
* The title of the video
|
||||
* @example '2018:E6 - GoldenEye: Source'
|
||||
*/
|
||||
title: string;
|
||||
/**
|
||||
* A description of the video. Maximum 2048 characters.
|
||||
* @example 'We play gun game in GoldenEye: Source with a good friend of ours. His name is Gruchy. Dan Gruchy.'
|
||||
*/
|
||||
description: string;
|
||||
/**
|
||||
* A URL pointing to the actual video media file. Should be one of the supported formats. HTML is not a supported
|
||||
* format. Flash is allowed, but no longer supported on most mobile platforms, and so may be indexed less well. Must
|
||||
* not be the same as the `<loc>` URL.
|
||||
* @example "http://streamserver.example.com/video123.mp4"
|
||||
*/
|
||||
content_loc?: string;
|
||||
/**
|
||||
* A URL pointing to a player for a specific video. Usually this is the information in the src element of an `<embed>`
|
||||
* tag. Must not be the same as the `<loc>` URL
|
||||
* @example "https://roosterteeth.com/embed/rouletsplay-2018-goldeneye-source"
|
||||
*/
|
||||
player_loc?: string;
|
||||
/**
|
||||
* A string the search engine can append as a query param to enable automatic
|
||||
* playback. Equivilant to auto play attr on player_loc tag.
|
||||
* @example 'ap=1'
|
||||
*/
|
||||
'player_loc:autoplay'?: string;
|
||||
/**
|
||||
* Whether the search engine can embed the video in search results. Allowed values are yes or no.
|
||||
*/
|
||||
'player_loc:allow_embed'?: EnumYesNo;
|
||||
/**
|
||||
* The length of the video in seconds
|
||||
* @example 600
|
||||
*/
|
||||
duration?: number;
|
||||
/**
|
||||
* The date after which the video will no longer be available.
|
||||
* @example "2012-07-16T19:20:30+08:00"
|
||||
*/
|
||||
expiration_date?: string;
|
||||
/**
|
||||
* The number of times the video has been viewed
|
||||
*/
|
||||
view_count?: number;
|
||||
/**
|
||||
* The date the video was first published, in W3C format.
|
||||
* @example "2012-07-16T19:20:30+08:00"
|
||||
*/
|
||||
publication_date?: string;
|
||||
/**
|
||||
* A short description of the broad category that the video belongs to. This is a string no longer than 256 characters.
|
||||
* @example Baking
|
||||
*/
|
||||
category?: string;
|
||||
/**
|
||||
* Whether to show or hide your video in search results from specific countries.
|
||||
* @example "IE GB US CA"
|
||||
*/
|
||||
restriction?: string;
|
||||
/**
|
||||
* Whether the countries in restriction are allowed or denied
|
||||
* @example 'deny'
|
||||
*/
|
||||
'restriction:relationship'?: EnumAllowDeny;
|
||||
gallery_loc?: string;
|
||||
/**
|
||||
* [Optional] Specifies the URL of a webpage with additional information about this uploader. This URL must be in the same domain as the <loc> tag.
|
||||
* @see https://developers.google.com/search/docs/advanced/sitemaps/video-sitemaps
|
||||
* @example http://www.example.com/users/grillymcgrillerson
|
||||
*/
|
||||
'uploader:info'?: string;
|
||||
'gallery_loc:title'?: string;
|
||||
/**
|
||||
* The price to download or view the video. Omit this tag for free videos.
|
||||
* @example "1.99"
|
||||
*/
|
||||
price?: string;
|
||||
/**
|
||||
* Specifies the resolution of the purchased version. Supported values are hd and sd.
|
||||
* @example "HD"
|
||||
*/
|
||||
'price:resolution'?: Resolution;
|
||||
/**
|
||||
* Specifies the currency in ISO4217 format.
|
||||
* @example "USD"
|
||||
*/
|
||||
'price:currency'?: string;
|
||||
/**
|
||||
* Specifies the purchase option. Supported values are rend and own.
|
||||
* @example "rent"
|
||||
*/
|
||||
'price:type'?: PriceType;
|
||||
/**
|
||||
* The video uploader's name. Only one <video:uploader> is allowed per video. String value, max 255 characters.
|
||||
* @example "GrillyMcGrillerson"
|
||||
*/
|
||||
uploader?: string;
|
||||
/**
|
||||
* Whether to show or hide your video in search results on specified platform types. This is a list of space-delimited
|
||||
* platform types. See <https://support.google.com/webmasters/answer/80471?hl=en&ref_topic=4581190> for more detail
|
||||
* @example "tv"
|
||||
*/
|
||||
platform?: string;
|
||||
id?: string;
|
||||
'platform:relationship'?: EnumAllowDeny;
|
||||
}
|
||||
export declare type PriceType = 'rent' | 'purchase' | 'RENT' | 'PURCHASE';
|
||||
export declare type Resolution = 'HD' | 'hd' | 'sd' | 'SD';
|
||||
/**
|
||||
* Sitemap video. <https://support.google.com/webmasters/answer/80471?hl=en&ref_topic=4581190>
|
||||
*/
|
||||
export interface VideoItem extends VideoItemBase {
|
||||
/**
|
||||
* An arbitrary string tag describing the video. Tags are generally very short descriptions of key concepts associated
|
||||
* with a video or piece of content.
|
||||
* @example ['Baking']
|
||||
*/
|
||||
tag: string[];
|
||||
/**
|
||||
* The rating of the video. Supported values are float numbers.
|
||||
* @example 2.5
|
||||
*/
|
||||
rating?: number;
|
||||
family_friendly?: EnumYesNo;
|
||||
/**
|
||||
* Indicates whether a subscription (either paid or free) is required to view
|
||||
* the video. Allowed values are yes or no.
|
||||
*/
|
||||
requires_subscription?: EnumYesNo;
|
||||
/**
|
||||
* Indicates whether the video is a live stream. Supported values are yes or no.
|
||||
*/
|
||||
live?: EnumYesNo;
|
||||
}
|
||||
/**
|
||||
* Sitemap video. <https://support.google.com/webmasters/answer/80471?hl=en&ref_topic=4581190>
|
||||
*/
|
||||
export interface VideoItemLoose extends VideoItemBase {
|
||||
/**
|
||||
* An arbitrary string tag describing the video. Tags are generally very short descriptions of key concepts associated
|
||||
* with a video or piece of content.
|
||||
* @example ['Baking']
|
||||
*/
|
||||
tag?: string | string[];
|
||||
/**
|
||||
* The rating of the video. Supported values are float numbers.
|
||||
* @example 2.5
|
||||
*/
|
||||
rating?: string | number;
|
||||
family_friendly?: EnumYesNo | boolean;
|
||||
requires_subscription?: EnumYesNo | boolean;
|
||||
/**
|
||||
* Indicates whether the video is a live stream. Supported values are yes or no.
|
||||
*/
|
||||
live?: EnumYesNo | boolean;
|
||||
}
|
||||
/**
|
||||
* https://support.google.com/webmasters/answer/189077
|
||||
*/
|
||||
export interface LinkItem {
|
||||
/**
|
||||
* @example 'en'
|
||||
*/
|
||||
lang: string;
|
||||
/**
|
||||
* @example 'en-us'
|
||||
*/
|
||||
hreflang?: string;
|
||||
url: string;
|
||||
}
|
||||
export interface IndexItem {
|
||||
url: string;
|
||||
lastmod?: string;
|
||||
}
|
||||
interface SitemapItemBase {
|
||||
lastmod?: string;
|
||||
changefreq?: EnumChangefreq;
|
||||
fullPrecisionPriority?: boolean;
|
||||
priority?: number;
|
||||
news?: NewsItem;
|
||||
expires?: string;
|
||||
androidLink?: string;
|
||||
ampLink?: string;
|
||||
url: string;
|
||||
}
|
||||
/**
|
||||
* Strict options for individual sitemap entries
|
||||
*/
|
||||
export interface SitemapItem extends SitemapItemBase {
|
||||
img: Img[];
|
||||
video: VideoItem[];
|
||||
links: LinkItem[];
|
||||
}
|
||||
/**
|
||||
* Options for individual sitemap entries prior to normalization
|
||||
*/
|
||||
export interface SitemapItemLoose extends SitemapItemBase {
|
||||
video?: VideoItemLoose | VideoItemLoose[];
|
||||
img?: string | Img | (string | Img)[];
|
||||
links?: LinkItem[];
|
||||
lastmodfile?: string | Buffer | URL;
|
||||
lastmodISO?: string;
|
||||
lastmodrealtime?: boolean;
|
||||
}
|
||||
/**
|
||||
* How to handle errors in passed in urls
|
||||
*/
|
||||
export declare enum ErrorLevel {
|
||||
/**
|
||||
* Validation will be skipped and nothing logged or thrown.
|
||||
*/
|
||||
SILENT = "silent",
|
||||
/**
|
||||
* If an invalid value is encountered, a console.warn will be called with details
|
||||
*/
|
||||
WARN = "warn",
|
||||
/**
|
||||
* An Error will be thrown on encountering invalid data.
|
||||
*/
|
||||
THROW = "throw"
|
||||
}
|
||||
export declare type ErrorHandler = (error: Error, level: ErrorLevel) => void;
|
||||
export declare enum TagNames {
|
||||
url = "url",
|
||||
loc = "loc",
|
||||
urlset = "urlset",
|
||||
lastmod = "lastmod",
|
||||
changefreq = "changefreq",
|
||||
priority = "priority",
|
||||
'video:thumbnail_loc' = "video:thumbnail_loc",
|
||||
'video:video' = "video:video",
|
||||
'video:title' = "video:title",
|
||||
'video:description' = "video:description",
|
||||
'video:tag' = "video:tag",
|
||||
'video:duration' = "video:duration",
|
||||
'video:player_loc' = "video:player_loc",
|
||||
'video:content_loc' = "video:content_loc",
|
||||
'image:image' = "image:image",
|
||||
'image:loc' = "image:loc",
|
||||
'image:geo_location' = "image:geo_location",
|
||||
'image:license' = "image:license",
|
||||
'image:title' = "image:title",
|
||||
'image:caption' = "image:caption",
|
||||
'video:requires_subscription' = "video:requires_subscription",
|
||||
'video:publication_date' = "video:publication_date",
|
||||
'video:id' = "video:id",
|
||||
'video:restriction' = "video:restriction",
|
||||
'video:family_friendly' = "video:family_friendly",
|
||||
'video:view_count' = "video:view_count",
|
||||
'video:uploader' = "video:uploader",
|
||||
'video:expiration_date' = "video:expiration_date",
|
||||
'video:platform' = "video:platform",
|
||||
'video:price' = "video:price",
|
||||
'video:rating' = "video:rating",
|
||||
'video:category' = "video:category",
|
||||
'video:live' = "video:live",
|
||||
'video:gallery_loc' = "video:gallery_loc",
|
||||
'news:news' = "news:news",
|
||||
'news:publication' = "news:publication",
|
||||
'news:name' = "news:name",
|
||||
'news:access' = "news:access",
|
||||
'news:genres' = "news:genres",
|
||||
'news:publication_date' = "news:publication_date",
|
||||
'news:title' = "news:title",
|
||||
'news:keywords' = "news:keywords",
|
||||
'news:stock_tickers' = "news:stock_tickers",
|
||||
'news:language' = "news:language",
|
||||
'mobile:mobile' = "mobile:mobile",
|
||||
'xhtml:link' = "xhtml:link",
|
||||
'expires' = "expires"
|
||||
}
|
||||
export declare enum IndexTagNames {
|
||||
sitemap = "sitemap",
|
||||
sitemapindex = "sitemapindex",
|
||||
loc = "loc",
|
||||
lastmod = "lastmod"
|
||||
}
|
||||
export {};
|
||||
143
node_modules/sitemap/dist/lib/types.js
generated
vendored
Normal file
143
node_modules/sitemap/dist/lib/types.js
generated
vendored
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.IndexTagNames = exports.TagNames = exports.ErrorLevel = exports.isAllowDeny = exports.EnumAllowDeny = exports.isValidYesNo = exports.EnumYesNo = exports.isValidChangeFreq = exports.CHANGEFREQ = exports.isResolution = exports.isPriceType = exports.validators = exports.EnumChangefreq = void 0;
|
||||
/**
|
||||
* How frequently the page is likely to change. This value provides general
|
||||
* information to search engines and may not correlate exactly to how often they crawl the page. Please note that the
|
||||
* value of this tag is considered a hint and not a command. See
|
||||
* <https://www.sitemaps.org/protocol.html#xmlTagDefinitions> for the acceptable
|
||||
* values
|
||||
*/
|
||||
var EnumChangefreq;
|
||||
(function (EnumChangefreq) {
|
||||
EnumChangefreq["DAILY"] = "daily";
|
||||
EnumChangefreq["MONTHLY"] = "monthly";
|
||||
EnumChangefreq["ALWAYS"] = "always";
|
||||
EnumChangefreq["HOURLY"] = "hourly";
|
||||
EnumChangefreq["WEEKLY"] = "weekly";
|
||||
EnumChangefreq["YEARLY"] = "yearly";
|
||||
EnumChangefreq["NEVER"] = "never";
|
||||
})(EnumChangefreq = exports.EnumChangefreq || (exports.EnumChangefreq = {}));
|
||||
const allowDeny = /^(?:allow|deny)$/;
|
||||
exports.validators = {
|
||||
'price:currency': /^[A-Z]{3}$/,
|
||||
'price:type': /^(?:rent|purchase|RENT|PURCHASE)$/,
|
||||
'price:resolution': /^(?:HD|hd|sd|SD)$/,
|
||||
'platform:relationship': allowDeny,
|
||||
'restriction:relationship': allowDeny,
|
||||
restriction: /^([A-Z]{2}( +[A-Z]{2})*)?$/,
|
||||
platform: /^((web|mobile|tv)( (web|mobile|tv))*)?$/,
|
||||
language: /^zh-cn|zh-tw|([a-z]{2,3})$/,
|
||||
genres: /^(PressRelease|Satire|Blog|OpEd|Opinion|UserGenerated)(, *(PressRelease|Satire|Blog|OpEd|Opinion|UserGenerated))*$/,
|
||||
stock_tickers: /^(\w+:\w+(, *\w+:\w+){0,4})?$/,
|
||||
};
|
||||
function isPriceType(pt) {
|
||||
return exports.validators['price:type'].test(pt);
|
||||
}
|
||||
exports.isPriceType = isPriceType;
|
||||
function isResolution(res) {
|
||||
return exports.validators['price:resolution'].test(res);
|
||||
}
|
||||
exports.isResolution = isResolution;
|
||||
exports.CHANGEFREQ = Object.values(EnumChangefreq);
|
||||
function isValidChangeFreq(freq) {
|
||||
return exports.CHANGEFREQ.includes(freq);
|
||||
}
|
||||
exports.isValidChangeFreq = isValidChangeFreq;
|
||||
var EnumYesNo;
|
||||
(function (EnumYesNo) {
|
||||
EnumYesNo["YES"] = "YES";
|
||||
EnumYesNo["NO"] = "NO";
|
||||
EnumYesNo["Yes"] = "Yes";
|
||||
EnumYesNo["No"] = "No";
|
||||
EnumYesNo["yes"] = "yes";
|
||||
EnumYesNo["no"] = "no";
|
||||
})(EnumYesNo = exports.EnumYesNo || (exports.EnumYesNo = {}));
|
||||
function isValidYesNo(yn) {
|
||||
return /^YES|NO|[Yy]es|[Nn]o$/.test(yn);
|
||||
}
|
||||
exports.isValidYesNo = isValidYesNo;
|
||||
var EnumAllowDeny;
|
||||
(function (EnumAllowDeny) {
|
||||
EnumAllowDeny["ALLOW"] = "allow";
|
||||
EnumAllowDeny["DENY"] = "deny";
|
||||
})(EnumAllowDeny = exports.EnumAllowDeny || (exports.EnumAllowDeny = {}));
|
||||
function isAllowDeny(ad) {
|
||||
return allowDeny.test(ad);
|
||||
}
|
||||
exports.isAllowDeny = isAllowDeny;
|
||||
/**
|
||||
* How to handle errors in passed in urls
|
||||
*/
|
||||
var ErrorLevel;
|
||||
(function (ErrorLevel) {
|
||||
/**
|
||||
* Validation will be skipped and nothing logged or thrown.
|
||||
*/
|
||||
ErrorLevel["SILENT"] = "silent";
|
||||
/**
|
||||
* If an invalid value is encountered, a console.warn will be called with details
|
||||
*/
|
||||
ErrorLevel["WARN"] = "warn";
|
||||
/**
|
||||
* An Error will be thrown on encountering invalid data.
|
||||
*/
|
||||
ErrorLevel["THROW"] = "throw";
|
||||
})(ErrorLevel = exports.ErrorLevel || (exports.ErrorLevel = {}));
|
||||
var TagNames;
|
||||
(function (TagNames) {
|
||||
TagNames["url"] = "url";
|
||||
TagNames["loc"] = "loc";
|
||||
TagNames["urlset"] = "urlset";
|
||||
TagNames["lastmod"] = "lastmod";
|
||||
TagNames["changefreq"] = "changefreq";
|
||||
TagNames["priority"] = "priority";
|
||||
TagNames["video:thumbnail_loc"] = "video:thumbnail_loc";
|
||||
TagNames["video:video"] = "video:video";
|
||||
TagNames["video:title"] = "video:title";
|
||||
TagNames["video:description"] = "video:description";
|
||||
TagNames["video:tag"] = "video:tag";
|
||||
TagNames["video:duration"] = "video:duration";
|
||||
TagNames["video:player_loc"] = "video:player_loc";
|
||||
TagNames["video:content_loc"] = "video:content_loc";
|
||||
TagNames["image:image"] = "image:image";
|
||||
TagNames["image:loc"] = "image:loc";
|
||||
TagNames["image:geo_location"] = "image:geo_location";
|
||||
TagNames["image:license"] = "image:license";
|
||||
TagNames["image:title"] = "image:title";
|
||||
TagNames["image:caption"] = "image:caption";
|
||||
TagNames["video:requires_subscription"] = "video:requires_subscription";
|
||||
TagNames["video:publication_date"] = "video:publication_date";
|
||||
TagNames["video:id"] = "video:id";
|
||||
TagNames["video:restriction"] = "video:restriction";
|
||||
TagNames["video:family_friendly"] = "video:family_friendly";
|
||||
TagNames["video:view_count"] = "video:view_count";
|
||||
TagNames["video:uploader"] = "video:uploader";
|
||||
TagNames["video:expiration_date"] = "video:expiration_date";
|
||||
TagNames["video:platform"] = "video:platform";
|
||||
TagNames["video:price"] = "video:price";
|
||||
TagNames["video:rating"] = "video:rating";
|
||||
TagNames["video:category"] = "video:category";
|
||||
TagNames["video:live"] = "video:live";
|
||||
TagNames["video:gallery_loc"] = "video:gallery_loc";
|
||||
TagNames["news:news"] = "news:news";
|
||||
TagNames["news:publication"] = "news:publication";
|
||||
TagNames["news:name"] = "news:name";
|
||||
TagNames["news:access"] = "news:access";
|
||||
TagNames["news:genres"] = "news:genres";
|
||||
TagNames["news:publication_date"] = "news:publication_date";
|
||||
TagNames["news:title"] = "news:title";
|
||||
TagNames["news:keywords"] = "news:keywords";
|
||||
TagNames["news:stock_tickers"] = "news:stock_tickers";
|
||||
TagNames["news:language"] = "news:language";
|
||||
TagNames["mobile:mobile"] = "mobile:mobile";
|
||||
TagNames["xhtml:link"] = "xhtml:link";
|
||||
TagNames["expires"] = "expires";
|
||||
})(TagNames = exports.TagNames || (exports.TagNames = {}));
|
||||
var IndexTagNames;
|
||||
(function (IndexTagNames) {
|
||||
IndexTagNames["sitemap"] = "sitemap";
|
||||
IndexTagNames["sitemapindex"] = "sitemapindex";
|
||||
IndexTagNames["loc"] = "loc";
|
||||
IndexTagNames["lastmod"] = "lastmod";
|
||||
})(IndexTagNames = exports.IndexTagNames || (exports.IndexTagNames = {}));
|
||||
55
node_modules/sitemap/dist/lib/utils.d.ts
generated
vendored
Normal file
55
node_modules/sitemap/dist/lib/utils.d.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
/// <reference types="node" />
|
||||
import { Readable, ReadableOptions, TransformOptions } from 'stream';
|
||||
import { SitemapItem, ErrorLevel, SitemapItemLoose, ErrorHandler } from './types';
|
||||
/**
|
||||
* Verifies all data passed in will comply with sitemap spec.
|
||||
* @param conf Options to validate
|
||||
* @param level logging level
|
||||
* @param errorHandler error handling func
|
||||
*/
|
||||
export declare function validateSMIOptions(conf: SitemapItem, level?: ErrorLevel, errorHandler?: ErrorHandler): SitemapItem;
|
||||
/**
|
||||
* Combines multiple streams into one
|
||||
* @param streams the streams to combine
|
||||
*/
|
||||
export declare function mergeStreams(streams: Readable[], options?: TransformOptions): Readable;
|
||||
export interface ReadlineStreamOptions extends ReadableOptions {
|
||||
input: Readable;
|
||||
}
|
||||
/**
|
||||
* Wraps node's ReadLine in a stream
|
||||
*/
|
||||
export declare class ReadlineStream extends Readable {
|
||||
private _source;
|
||||
constructor(options: ReadlineStreamOptions);
|
||||
_read(size: number): void;
|
||||
}
|
||||
/**
|
||||
* Takes a stream likely from fs.createReadStream('./path') and returns a stream
|
||||
* of sitemap items
|
||||
* @param stream a stream of line separated urls.
|
||||
* @param opts.isJSON is the stream line separated JSON. leave undefined to guess
|
||||
*/
|
||||
export declare function lineSeparatedURLsToSitemapOptions(stream: Readable, { isJSON }?: {
|
||||
isJSON?: boolean;
|
||||
}): Readable;
|
||||
/**
|
||||
* Based on lodash's implementation of chunk.
|
||||
*
|
||||
* Copyright JS Foundation and other contributors <https://js.foundation/>
|
||||
*
|
||||
* Based on Underscore.js, copyright Jeremy Ashkenas,
|
||||
* DocumentCloud and Investigative Reporters & Editors <http://underscorejs.org/>
|
||||
*
|
||||
* This software consists of voluntary contributions made by many
|
||||
* individuals. For exact contribution history, see the revision history
|
||||
* available at https://github.com/lodash/lodash
|
||||
*/
|
||||
export declare function chunk(array: any[], size?: number): any[];
|
||||
/**
|
||||
* Converts the passed in sitemap entry into one capable of being consumed by SitemapItem
|
||||
* @param {string | SitemapItemLoose} elem the string or object to be converted
|
||||
* @param {string} hostname
|
||||
* @returns SitemapItemOptions a strict sitemap item option
|
||||
*/
|
||||
export declare function normalizeURL(elem: string | SitemapItemLoose, hostname?: string, lastmodDateOnly?: boolean): SitemapItem;
|
||||
353
node_modules/sitemap/dist/lib/utils.js
generated
vendored
Normal file
353
node_modules/sitemap/dist/lib/utils.js
generated
vendored
Normal file
|
|
@ -0,0 +1,353 @@
|
|||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.normalizeURL = exports.chunk = exports.lineSeparatedURLsToSitemapOptions = exports.ReadlineStream = exports.mergeStreams = exports.validateSMIOptions = void 0;
|
||||
/*!
|
||||
* Sitemap
|
||||
* Copyright(c) 2011 Eugene Kalinin
|
||||
* MIT Licensed
|
||||
*/
|
||||
const fs_1 = require("fs");
|
||||
const stream_1 = require("stream");
|
||||
const readline_1 = require("readline");
|
||||
const url_1 = require("url");
|
||||
const types_1 = require("./types");
|
||||
const errors_1 = require("./errors");
|
||||
const types_2 = require("./types");
|
||||
function validate(subject, name, url, level) {
|
||||
Object.keys(subject).forEach((key) => {
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
const val = subject[key];
|
||||
if (types_2.validators[key] && !types_2.validators[key].test(val)) {
|
||||
if (level === types_1.ErrorLevel.THROW) {
|
||||
throw new errors_1.InvalidAttrValue(key, val, types_2.validators[key]);
|
||||
}
|
||||
else {
|
||||
console.warn(`${url}: ${name} key ${key} has invalid value: ${val}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
function handleError(error, level) {
|
||||
if (level === types_1.ErrorLevel.THROW) {
|
||||
throw error;
|
||||
}
|
||||
else if (level === types_1.ErrorLevel.WARN) {
|
||||
console.warn(error.name, error.message);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Verifies all data passed in will comply with sitemap spec.
|
||||
* @param conf Options to validate
|
||||
* @param level logging level
|
||||
* @param errorHandler error handling func
|
||||
*/
|
||||
function validateSMIOptions(conf, level = types_1.ErrorLevel.WARN, errorHandler = handleError) {
|
||||
if (!conf) {
|
||||
throw new errors_1.NoConfigError();
|
||||
}
|
||||
if (level === types_1.ErrorLevel.SILENT) {
|
||||
return conf;
|
||||
}
|
||||
const { url, changefreq, priority, news, video } = conf;
|
||||
if (!url) {
|
||||
errorHandler(new errors_1.NoURLError(), level);
|
||||
}
|
||||
if (changefreq) {
|
||||
if (!(0, types_1.isValidChangeFreq)(changefreq)) {
|
||||
errorHandler(new errors_1.ChangeFreqInvalidError(url, changefreq), level);
|
||||
}
|
||||
}
|
||||
if (priority) {
|
||||
if (!(priority >= 0.0 && priority <= 1.0)) {
|
||||
errorHandler(new errors_1.PriorityInvalidError(url, priority), level);
|
||||
}
|
||||
}
|
||||
if (news) {
|
||||
if (news.access &&
|
||||
news.access !== 'Registration' &&
|
||||
news.access !== 'Subscription') {
|
||||
errorHandler(new errors_1.InvalidNewsAccessValue(url, news.access), level);
|
||||
}
|
||||
if (!news.publication ||
|
||||
!news.publication.name ||
|
||||
!news.publication.language ||
|
||||
!news.publication_date ||
|
||||
!news.title) {
|
||||
errorHandler(new errors_1.InvalidNewsFormat(url), level);
|
||||
}
|
||||
validate(news, 'news', url, level);
|
||||
validate(news.publication, 'publication', url, level);
|
||||
}
|
||||
if (video) {
|
||||
video.forEach((vid) => {
|
||||
var _a;
|
||||
if (vid.duration !== undefined) {
|
||||
if (vid.duration < 0 || vid.duration > 28800) {
|
||||
errorHandler(new errors_1.InvalidVideoDuration(url, vid.duration), level);
|
||||
}
|
||||
}
|
||||
if (vid.rating !== undefined && (vid.rating < 0 || vid.rating > 5)) {
|
||||
errorHandler(new errors_1.InvalidVideoRating(url, vid.title, vid.rating), level);
|
||||
}
|
||||
if (typeof vid !== 'object' ||
|
||||
!vid.thumbnail_loc ||
|
||||
!vid.title ||
|
||||
!vid.description) {
|
||||
// has to be an object and include required categories https://support.google.com/webmasters/answer/80471?hl=en&ref_topic=4581190
|
||||
errorHandler(new errors_1.InvalidVideoFormat(url), level);
|
||||
}
|
||||
if (vid.title.length > 100) {
|
||||
errorHandler(new errors_1.InvalidVideoTitle(url, vid.title.length), level);
|
||||
}
|
||||
if (vid.description.length > 2048) {
|
||||
errorHandler(new errors_1.InvalidVideoDescription(url, vid.description.length), level);
|
||||
}
|
||||
if (vid.view_count !== undefined && vid.view_count < 0) {
|
||||
errorHandler(new errors_1.InvalidVideoViewCount(url, vid.view_count), level);
|
||||
}
|
||||
if (vid.tag.length > 32) {
|
||||
errorHandler(new errors_1.InvalidVideoTagCount(url, vid.tag.length), level);
|
||||
}
|
||||
if (vid.category !== undefined && ((_a = vid.category) === null || _a === void 0 ? void 0 : _a.length) > 256) {
|
||||
errorHandler(new errors_1.InvalidVideoCategory(url, vid.category.length), level);
|
||||
}
|
||||
if (vid.family_friendly !== undefined &&
|
||||
!(0, types_1.isValidYesNo)(vid.family_friendly)) {
|
||||
errorHandler(new errors_1.InvalidVideoFamilyFriendly(url, vid.family_friendly), level);
|
||||
}
|
||||
if (vid.restriction) {
|
||||
if (!types_2.validators.restriction.test(vid.restriction)) {
|
||||
errorHandler(new errors_1.InvalidVideoRestriction(url, vid.restriction), level);
|
||||
}
|
||||
if (!vid['restriction:relationship'] ||
|
||||
!(0, types_1.isAllowDeny)(vid['restriction:relationship'])) {
|
||||
errorHandler(new errors_1.InvalidVideoRestrictionRelationship(url, vid['restriction:relationship']), level);
|
||||
}
|
||||
}
|
||||
// TODO price element should be unbounded
|
||||
if ((vid.price === '' && vid['price:type'] === undefined) ||
|
||||
(vid['price:type'] !== undefined && !(0, types_1.isPriceType)(vid['price:type']))) {
|
||||
errorHandler(new errors_1.InvalidVideoPriceType(url, vid['price:type'], vid.price), level);
|
||||
}
|
||||
if (vid['price:resolution'] !== undefined &&
|
||||
!(0, types_1.isResolution)(vid['price:resolution'])) {
|
||||
errorHandler(new errors_1.InvalidVideoResolution(url, vid['price:resolution']), level);
|
||||
}
|
||||
if (vid['price:currency'] !== undefined &&
|
||||
!types_2.validators['price:currency'].test(vid['price:currency'])) {
|
||||
errorHandler(new errors_1.InvalidVideoPriceCurrency(url, vid['price:currency']), level);
|
||||
}
|
||||
validate(vid, 'video', url, level);
|
||||
});
|
||||
}
|
||||
return conf;
|
||||
}
|
||||
exports.validateSMIOptions = validateSMIOptions;
|
||||
/**
|
||||
* Combines multiple streams into one
|
||||
* @param streams the streams to combine
|
||||
*/
|
||||
function mergeStreams(streams, options) {
|
||||
let pass = new stream_1.PassThrough(options);
|
||||
let waiting = streams.length;
|
||||
for (const stream of streams) {
|
||||
pass = stream.pipe(pass, { end: false });
|
||||
stream.once('end', () => --waiting === 0 && pass.emit('end'));
|
||||
}
|
||||
return pass;
|
||||
}
|
||||
exports.mergeStreams = mergeStreams;
|
||||
/**
|
||||
* Wraps node's ReadLine in a stream
|
||||
*/
|
||||
class ReadlineStream extends stream_1.Readable {
|
||||
constructor(options) {
|
||||
if (options.autoDestroy === undefined) {
|
||||
options.autoDestroy = true;
|
||||
}
|
||||
options.objectMode = true;
|
||||
super(options);
|
||||
this._source = (0, readline_1.createInterface)({
|
||||
input: options.input,
|
||||
terminal: false,
|
||||
crlfDelay: Infinity,
|
||||
});
|
||||
// Every time there's data, push it into the internal buffer.
|
||||
this._source.on('line', (chunk) => {
|
||||
// If push() returns false, then stop reading from source.
|
||||
if (!this.push(chunk))
|
||||
this._source.pause();
|
||||
});
|
||||
// When the source ends, push the EOF-signaling `null` chunk.
|
||||
this._source.on('close', () => {
|
||||
this.push(null);
|
||||
});
|
||||
}
|
||||
// _read() will be called when the stream wants to pull more data in.
|
||||
// The advisory size argument is ignored in this case.
|
||||
_read(size) {
|
||||
this._source.resume();
|
||||
}
|
||||
}
|
||||
exports.ReadlineStream = ReadlineStream;
|
||||
/**
|
||||
* Takes a stream likely from fs.createReadStream('./path') and returns a stream
|
||||
* of sitemap items
|
||||
* @param stream a stream of line separated urls.
|
||||
* @param opts.isJSON is the stream line separated JSON. leave undefined to guess
|
||||
*/
|
||||
function lineSeparatedURLsToSitemapOptions(stream, { isJSON } = {}) {
|
||||
return new ReadlineStream({ input: stream }).pipe(new stream_1.Transform({
|
||||
objectMode: true,
|
||||
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
|
||||
transform: (line, encoding, cb) => {
|
||||
if (isJSON || (isJSON === undefined && line[0] === '{')) {
|
||||
cb(null, JSON.parse(line));
|
||||
}
|
||||
else {
|
||||
cb(null, line);
|
||||
}
|
||||
},
|
||||
}));
|
||||
}
|
||||
exports.lineSeparatedURLsToSitemapOptions = lineSeparatedURLsToSitemapOptions;
|
||||
/**
|
||||
* Based on lodash's implementation of chunk.
|
||||
*
|
||||
* Copyright JS Foundation and other contributors <https://js.foundation/>
|
||||
*
|
||||
* Based on Underscore.js, copyright Jeremy Ashkenas,
|
||||
* DocumentCloud and Investigative Reporters & Editors <http://underscorejs.org/>
|
||||
*
|
||||
* This software consists of voluntary contributions made by many
|
||||
* individuals. For exact contribution history, see the revision history
|
||||
* available at https://github.com/lodash/lodash
|
||||
*/
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
function chunk(array, size = 1) {
|
||||
size = Math.max(Math.trunc(size), 0);
|
||||
const length = array ? array.length : 0;
|
||||
if (!length || size < 1) {
|
||||
return [];
|
||||
}
|
||||
const result = Array(Math.ceil(length / size));
|
||||
let index = 0, resIndex = 0;
|
||||
while (index < length) {
|
||||
result[resIndex++] = array.slice(index, (index += size));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
exports.chunk = chunk;
|
||||
function boolToYESNO(bool) {
|
||||
if (bool === undefined) {
|
||||
return bool;
|
||||
}
|
||||
if (typeof bool === 'boolean') {
|
||||
return bool ? types_1.EnumYesNo.yes : types_1.EnumYesNo.no;
|
||||
}
|
||||
return bool;
|
||||
}
|
||||
/**
|
||||
* Converts the passed in sitemap entry into one capable of being consumed by SitemapItem
|
||||
* @param {string | SitemapItemLoose} elem the string or object to be converted
|
||||
* @param {string} hostname
|
||||
* @returns SitemapItemOptions a strict sitemap item option
|
||||
*/
|
||||
function normalizeURL(elem, hostname, lastmodDateOnly = false) {
|
||||
// SitemapItem
|
||||
// create object with url property
|
||||
let smi = {
|
||||
img: [],
|
||||
video: [],
|
||||
links: [],
|
||||
url: '',
|
||||
};
|
||||
let smiLoose;
|
||||
if (typeof elem === 'string') {
|
||||
smi.url = elem;
|
||||
smiLoose = { url: elem };
|
||||
}
|
||||
else {
|
||||
smiLoose = elem;
|
||||
}
|
||||
smi.url = new url_1.URL(smiLoose.url, hostname).toString();
|
||||
let img = [];
|
||||
if (smiLoose.img) {
|
||||
if (typeof smiLoose.img === 'string') {
|
||||
// string -> array of objects
|
||||
smiLoose.img = [{ url: smiLoose.img }];
|
||||
}
|
||||
else if (!Array.isArray(smiLoose.img)) {
|
||||
// object -> array of objects
|
||||
smiLoose.img = [smiLoose.img];
|
||||
}
|
||||
img = smiLoose.img.map((el) => (typeof el === 'string' ? { url: el } : el));
|
||||
}
|
||||
// prepend hostname to all image urls
|
||||
smi.img = img.map((el) => ({
|
||||
...el,
|
||||
url: new url_1.URL(el.url, hostname).toString(),
|
||||
}));
|
||||
let links = [];
|
||||
if (smiLoose.links) {
|
||||
links = smiLoose.links;
|
||||
}
|
||||
smi.links = links.map((link) => {
|
||||
return { ...link, url: new url_1.URL(link.url, hostname).toString() };
|
||||
});
|
||||
if (smiLoose.video) {
|
||||
if (!Array.isArray(smiLoose.video)) {
|
||||
// make it an array
|
||||
smiLoose.video = [smiLoose.video];
|
||||
}
|
||||
smi.video = smiLoose.video.map((video) => {
|
||||
const nv = {
|
||||
...video,
|
||||
family_friendly: boolToYESNO(video.family_friendly),
|
||||
live: boolToYESNO(video.live),
|
||||
requires_subscription: boolToYESNO(video.requires_subscription),
|
||||
tag: [],
|
||||
rating: undefined,
|
||||
};
|
||||
if (video.tag !== undefined) {
|
||||
nv.tag = !Array.isArray(video.tag) ? [video.tag] : video.tag;
|
||||
}
|
||||
if (video.rating !== undefined) {
|
||||
if (typeof video.rating === 'string') {
|
||||
nv.rating = parseFloat(video.rating);
|
||||
}
|
||||
else {
|
||||
nv.rating = video.rating;
|
||||
}
|
||||
}
|
||||
if (typeof video.view_count === 'string') {
|
||||
nv.view_count = parseInt(video.view_count, 10);
|
||||
}
|
||||
else if (typeof video.view_count === 'number') {
|
||||
nv.view_count = video.view_count;
|
||||
}
|
||||
return nv;
|
||||
});
|
||||
}
|
||||
// If given a file to use for last modified date
|
||||
if (smiLoose.lastmodfile) {
|
||||
const { mtime } = (0, fs_1.statSync)(smiLoose.lastmodfile);
|
||||
smi.lastmod = new Date(mtime).toISOString();
|
||||
// The date of last modification (YYYY-MM-DD)
|
||||
}
|
||||
else if (smiLoose.lastmodISO) {
|
||||
smi.lastmod = new Date(smiLoose.lastmodISO).toISOString();
|
||||
}
|
||||
else if (smiLoose.lastmod) {
|
||||
smi.lastmod = new Date(smiLoose.lastmod).toISOString();
|
||||
}
|
||||
if (lastmodDateOnly && smi.lastmod) {
|
||||
smi.lastmod = smi.lastmod.slice(0, 10);
|
||||
}
|
||||
delete smiLoose.lastmodfile;
|
||||
delete smiLoose.lastmodISO;
|
||||
smi = { ...smiLoose, ...smi };
|
||||
return smi;
|
||||
}
|
||||
exports.normalizeURL = normalizeURL;
|
||||
8
node_modules/sitemap/dist/lib/xmllint.d.ts
generated
vendored
Normal file
8
node_modules/sitemap/dist/lib/xmllint.d.ts
generated
vendored
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
/// <reference types="node" />
|
||||
import { Readable } from 'stream';
|
||||
/**
|
||||
* Verify the passed in xml is valid. Requires xmllib be installed
|
||||
* @param xml what you want validated
|
||||
* @return {Promise<void>} resolves on valid rejects [error stderr]
|
||||
*/
|
||||
export declare function xmlLint(xml: string | Readable): Promise<void>;
|
||||
43
node_modules/sitemap/dist/lib/xmllint.js
generated
vendored
Normal file
43
node_modules/sitemap/dist/lib/xmllint.js
generated
vendored
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.xmlLint = void 0;
|
||||
const path_1 = require("path");
|
||||
const child_process_1 = require("child_process");
|
||||
const errors_1 = require("./errors");
|
||||
/**
|
||||
* Verify the passed in xml is valid. Requires xmllib be installed
|
||||
* @param xml what you want validated
|
||||
* @return {Promise<void>} resolves on valid rejects [error stderr]
|
||||
*/
|
||||
function xmlLint(xml) {
|
||||
const args = [
|
||||
'--schema',
|
||||
(0, path_1.resolve)(__dirname, '..', '..', 'schema', 'all.xsd'),
|
||||
'--noout',
|
||||
'-',
|
||||
];
|
||||
if (typeof xml === 'string') {
|
||||
args[args.length - 1] = xml;
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
(0, child_process_1.execFile)('which', ['xmllint'], (error, stdout, stderr) => {
|
||||
if (error) {
|
||||
reject([new errors_1.XMLLintUnavailable()]);
|
||||
return;
|
||||
}
|
||||
const xmllint = (0, child_process_1.execFile)('xmllint', args, (error, stdout, stderr) => {
|
||||
if (error) {
|
||||
reject([error, stderr]);
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
if (xmllint.stdout) {
|
||||
xmllint.stdout.unpipe();
|
||||
if (typeof xml !== 'string' && xml && xmllint.stdin) {
|
||||
xml.pipe(xmllint.stdin);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
exports.xmlLint = xmlLint;
|
||||
Loading…
Add table
Add a link
Reference in a new issue