Compare commits

...

2 Commits

Author SHA1 Message Date
Drake 35b13cac0d slight http session cleanup + alternate instance test
ci/woodpecker/push/woodpecker Pipeline failed Details
1 year ago
Drake 357097ab7d make chapters manually inited
1 year ago

@ -4,4 +4,5 @@ export default AO3;
import Chapter from "./src/classes/Chapter.ts"; import Chapter from "./src/classes/Chapter.ts";
import Search from "./src/classes/Search.ts"; import Search from "./src/classes/Search.ts";
import Work from "./src/classes/Work.ts"; import Work from "./src/classes/Work.ts";
export { AO3, Chapter, Search, Work }; const VERSION = "v0.2.0";
export { AO3, Chapter, Search, VERSION, Work };

@ -3,73 +3,23 @@ import { ID } from "../types.d.ts";
import { DOMParser } from "https://deno.land/x/deno_dom@v0.1.36-alpha/deno-dom-wasm.ts"; import { DOMParser } from "https://deno.land/x/deno_dom@v0.1.36-alpha/deno-dom-wasm.ts";
import { HTMLDocument } from "../types.d.ts"; import { HTMLDocument } from "../types.d.ts";
import Search, { SearchParameters } from "./Search.ts"; import Search, { SearchParameters } from "./Search.ts";
import { import { newSession, Options } from "../utils/http.ts";
CookieJar,
wrapFetch,
} from "https://deno.land/x/another_cookiejar@v5.0.1/mod.ts";
export default class AO3 { export default class AO3 {
session: { session: ReturnType<typeof newSession>;
get: (path: string) => Promise<Response>;
post: (
path: string,
payload: unknown,
) => Promise<Response>;
};
DOMParser = new DOMParser(); DOMParser = new DOMParser();
fetch: typeof fetch;
cookieJar: CookieJar;
#headers: Record<string, string>;
/** /**
* a representation of AO3 in class form * a representation of AO3 in class form
*/ */
constructor(opts?: { constructor(opts?: Options) {
url?: string; if (opts && !opts.headers) {
}) { opts.headers = {
this.cookieJar = new CookieJar(); "User-Agent":
this.fetch = wrapFetch({ cookieJar: this.cookieJar }); "Mozilla/5.0 (Windows NT 10.0; rv:108.0) Gecko/20100101 Firefox/108.0",
this.#headers = { };
"User-Agent": }
"Mozilla/5.0 (Windows NT 10.0; rv:108.0) Gecko/20100101 Firefox/108.0", this.session = newSession(opts ?? {});
};
this.session = {
get: async (path: string) => {
const res = await this.fetch(
opts?.url ?? "https://archiveofourown.org/" + path,
{
headers: this.#headers,
},
);
if (res.status > 300) {
console.log(res);
throw new Error("Failed request, probably rate-limited");
}
return res;
},
post: async (
path: string,
// deno-lint-ignore no-explicit-any
payload: any,
headers?: string,
) => {
const res = await this.fetch(
opts?.url ?? "https://archiveofourown.org/" + path,
{
"credentials": "include",
headers: Object.assign(headers ?? {}, this.#headers),
method: "POST",
body: payload,
},
);
if (res.status > 300) {
console.log(res);
throw new Error("Failed request, probably rate-limited");
}
return res;
},
};
} }
/** /**

@ -4,26 +4,16 @@ export default class Chapter {
#session: { #session: {
get: (path: string) => Promise<Response>; get: (path: string) => Promise<Response>;
}; };
isInited = false;
#document!: HTMLDocument; #document!: HTMLDocument;
#DOMParser: DOMParser; #DOMParser: DOMParser;
#id: ID; id: ID;
#workID: ID; workID: ID;
#name!: string; name!: string;
#html!: string; html!: string;
#text!: string; text!: string;
#summary!: string; summary!: string;
#startNote!: string; startNote!: string;
#endNote!: string; endNote!: string;
earlyName?: Promise<string>;
id!: Promise<ID>;
workID!: Promise<ID>;
name!: Promise<string>;
html!: Promise<string>;
text!: Promise<string>;
summary!: Promise<string>;
startNote!: Promise<string>;
endNote!: Promise<string>;
constructor( constructor(
workId: ID, workId: ID,
@ -36,45 +26,15 @@ export default class Chapter {
extraInfo: Record<string, any>, extraInfo: Record<string, any>,
) { ) {
this.#session = session; this.#session = session;
this.#workID = workId; this.workID = workId;
this.#id = id; this.id = id;
this.#DOMParser = DOMParser; this.#DOMParser = DOMParser;
this.earlyName = extraInfo.name; this.name = extraInfo.name;
return new Proxy(this, {
get: async (target, prop) => {
if (prop === "earlyName") {
return this.earlyName;
}
if (!this.isInited) {
await target.init();
target.isInited = true;
}
switch (prop) {
case "id":
return target.#id;
case "workID":
return target.#workID;
case "name":
return target.#name;
case "html":
return target.#html;
case "text":
return target.#text;
case "summary":
return target.#summary;
case "startNote":
return target.#startNote;
case "endNote":
return target.#endNote;
}
},
});
} }
async init() { async init() {
const res = await this.#session.get( const res = await this.#session.get(
`/works/${this.#workID}/chapters/${this.#id}?view_adult=true`, `/works/${this.workID}/chapters/${this.id}?view_adult=true`,
); );
this.#document = this.#DOMParser.parseFromString( this.#document = this.#DOMParser.parseFromString(
await res.text(), await res.text(),
@ -93,7 +53,7 @@ export default class Chapter {
} }
populateMetadata() { populateMetadata() {
this.#name = this.#document.querySelector("h3.title")?.innerText this.name = this.#document.querySelector("h3.title")?.innerText
.replace( .replace(
/Chapter \d+: /, /Chapter \d+: /,
"", "",
@ -101,7 +61,7 @@ export default class Chapter {
} }
populateSummary() { populateSummary() {
this.#summary = this.#document.querySelector("#summary > .userstuff") this.summary = this.#document.querySelector("#summary > .userstuff")
?.innerText.trim() as string; ?.innerText.trim() as string;
} }
@ -109,14 +69,14 @@ export default class Chapter {
const notesList = Array.from( const notesList = Array.from(
this.#document.querySelectorAll(".notes > .userstuff"), this.#document.querySelectorAll(".notes > .userstuff"),
).map((n) => (n as Element).innerHTML); ).map((n) => (n as Element).innerHTML);
this.#startNote = notesList[0]?.trim()?.replace(/<\/{0,1}p>/g, "\n") this.startNote = notesList[0]?.trim()?.replace(/<\/{0,1}p>/g, "\n")
?.trim(); ?.trim();
this.#endNote = notesList[1]?.trim()?.replace(/<\/{0,1}p>/g, "\n") this.endNote = notesList[1]?.trim()?.replace(/<\/{0,1}p>/g, "\n")
?.trim(); ?.trim();
} }
async populateText() { async populateText() {
this.#text = ""; this.text = "";
const elements = this.#document.querySelectorAll( const elements = this.#document.querySelectorAll(
"div.userstuff[role='article'] > p", "div.userstuff[role='article'] > p",
@ -125,25 +85,25 @@ export default class Chapter {
for (let i = 0; i < elements.length; i++) { for (let i = 0; i < elements.length; i++) {
const element = elements[i] as Element; const element = elements[i] as Element;
this.#text += element.innerText + "\n"; this.text += element.innerText + "\n";
} }
try { try {
this.#text = this.#text.trim(); this.text = this.text.trim();
this.#html = (this.#document.querySelector( this.html = (this.#document.querySelector(
"div.userstuff[role='article']", "div.userstuff[role='article']",
) as Element).innerHTML; ) as Element).innerHTML;
} catch { } catch {
//assume single chapter work //assume single chapter work
const res = await this.#session.get( const res = await this.#session.get(
`/works/${this.#workID}?view_adult=true`, `/works/${this.workID}?view_adult=true`,
); );
this.#document = this.#DOMParser.parseFromString( this.#document = this.#DOMParser.parseFromString(
await res.text(), await res.text(),
"text/html", "text/html",
) as HTMLDocument; ) as HTMLDocument;
this.#html = (this.#document.querySelector( this.html = (this.#document.querySelector(
"[role='article'] > div.userstuff", "[role='article'] > div.userstuff",
) as Element).innerHTML; ) as Element).innerHTML;
@ -154,8 +114,8 @@ export default class Chapter {
for (let i = 0; i < elements.length; i++) { for (let i = 0; i < elements.length; i++) {
const element = elements[i] as Element; const element = elements[i] as Element;
this.#text += element.innerText + "\n"; this.text += element.innerText + "\n";
this.#html += element.innerHTML; this.html += element.innerHTML;
} }
} }
} }

@ -75,19 +75,6 @@ export default class Work {
} }
populateTags() { populateTags() {
/* this.#document.querySelectorAll("dd.fandom > ul.commas > li").map(
(t) => this.tags.push(t.text),
);
this.#document.querySelectorAll("dd.relationship > ul.commas > li").map(
(t) => this.tags.push(t.text),
);
this.#document.querySelectorAll("dd.character > ul.commas > li").map(
(t) => this.tags.push(t.text),
);
this.#document.querySelectorAll("dd.freeform > ul.commas > li").map(
(t) => this.tags.push(t.text),
); */
const elements = this.#document.querySelectorAll("dd > ul.commas > li"); const elements = this.#document.querySelectorAll("dd > ul.commas > li");
for (let i = 0; i < elements.length; i++) { for (let i = 0; i < elements.length; i++) {

@ -0,0 +1,66 @@
// custom fetch wrappers.. for reasons
import {
CookieJar,
wrapFetch,
} from "https://deno.land/x/another_cookiejar@v5.0.1/mod.ts";
const cookieJar = new CookieJar();
const wrappedFetch = wrapFetch({
cookieJar,
});
const defaultOptions = {
url: "https://archiveofourown.org/",
headers: {
"User-Agent":
`Mozilla/5.0 (Windows NT 10.0; rv:108.0) Gecko/20100101 Firefox/108.0`,
},
};
export interface Options {
url?: string;
headers?: Record<string, string>;
}
const newSession = (opts: Options) => {
opts = Object.assign(defaultOptions, opts);
return {
get: async (path: string) => {
const res = await wrappedFetch(
opts.url + path,
{
headers: opts.headers,
},
);
if (res.status > 300) {
console.log(res);
throw new Error("Failed request, probably rate-limited");
}
return res;
},
post: async (
path: string,
// deno-lint-ignore no-explicit-any
payload: any,
headers?: string,
) => {
const res = await wrappedFetch(
opts.url + path,
{
"credentials": "include",
headers: Object.assign(headers ?? {}, opts.headers),
method: "POST",
body: payload,
},
);
if (res.status > 300) {
console.log(res);
throw new Error("Failed request, probably rate-limited");
}
return res;
},
};
};
export { newSession };

@ -1,40 +1,56 @@
//FIXME: we need to test single-chapter works too (because those seem to be really inconsistent for some reason?) //FIXME: we need to test single-chapter works too (because those seem to be really inconsistent for some reason?)
import AO3 from "../mod.ts"; import AO3 from "../mod.ts";
import { assert } from "https://deno.land/std@0.167.0/testing/asserts.ts"; import {
assert,
AssertionError,
} from "https://deno.land/std@0.167.0/testing/asserts.ts";
export default function test(ao3: AO3) { export default function test(ao3: AO3) {
Deno.test("chapters", async (test) => { Deno.test("chapters", async (test) => {
const work = await ao3.getWork("43251729"); const work = await ao3.getWork("43251729");
await work.init(); await work.init();
assert(work.chapters.length > 0, "chapters array is not initialized"); await test.step("initialization", async () => {
await test.step("IDs", async () => {
assert( assert(
await work.chapters[0].id === "108714198", work.chapters.length > 0,
"chapters array is not initialized",
);
try {
await work.chapters[0].init();
await work.chapters[3].init();
} catch {
throw new AssertionError("failed to initialize chapter");
}
});
await test.step("IDs", () => {
assert(
work.chapters[0].id === "108714198",
"incorrect chapter ID", "incorrect chapter ID",
); );
assert( assert(
await work.chapters[0].workID === "43251729", work.chapters[0].workID === "43251729",
"incorrect work ID", "incorrect work ID",
); //why do we even store the work's ID publicly in a chapter? ); //why do we even store the work's ID publicly in a chapter?
}); });
await test.step("name", async () => { await test.step("name", () => {
assert( assert(
await work.chapters[0].name === "Welcome to the Studio", work.chapters[0].name === "Welcome to the Studio",
"incorrect/missing chapter names", "incorrect/missing chapter names",
); );
}); });
await test.step("content", async () => { await test.step("content", () => {
//FIXME: this should probably be tested better //FIXME: this should probably be tested better
assert( assert(
(await work.chapters[0].text).length > 0, (work.chapters[0].text).length > 0,
"text/content is completely missing", "text/content is completely missing",
); );
}); });
await test.step("notes and summary", async () => { await test.step("notes and summary", () => {
//FIXME: write a chapter of my fic (lol) that includes a summary and end note for testing //FIXME: write a chapter of my fic (lol) that includes a summary and end note for testing
assert( assert(
await work.chapters[3].startNote === work.chapters[3].startNote ===
`If you haven't noticed yet, most of these chapters are named after Bendy fansongs `If you haven't noticed yet, most of these chapters are named after Bendy fansongs
This is definitely because I'm trying to be smart and cool and make funny references, and definitely not because I'm uncreative :)`, This is definitely because I'm trying to be smart and cool and make funny references, and definitely not because I'm uncreative :)`,

@ -2,9 +2,11 @@ import AO3 from "../src/classes/AO3.ts";
import workTest from "./work.ts"; import workTest from "./work.ts";
import chaptersTest from "./chapter.ts"; import chaptersTest from "./chapter.ts";
import searchTest from "./search.ts"; import searchTest from "./search.ts";
import miscTest from "./misc.ts";
const ao3 = new AO3(); const ao3 = new AO3();
await workTest(ao3); await workTest(ao3);
await chaptersTest(ao3); await chaptersTest(ao3);
await searchTest(ao3); await searchTest(ao3);
await miscTest(ao3);

@ -0,0 +1,20 @@
import AO3 from "../mod.ts";
import { assert } from "https://deno.land/std@0.167.0/testing/asserts.ts";
export default function test(ao3: AO3) {
Deno.test("misc", async (test) => {
await test.step("alternate instances", async () => {
const ao3 = new AO3({
url: "https://squidgeworld.org/",
});
const work = await ao3.getWork(34491);
await work.init();
assert(
work.name === "Implementing OTW's Code To Build SquidgeWorld",
"failed to get a work from alternate instance",
);
});
});
}

@ -1,5 +1,4 @@
import AO3 from "../mod.ts"; import AO3 from "../mod.ts";
import type { Search } from "../mod.ts";
import { assert } from "https://deno.land/std@0.167.0/testing/asserts.ts"; import { assert } from "https://deno.land/std@0.167.0/testing/asserts.ts";
export default function test(ao3: AO3) { export default function test(ao3: AO3) {

Loading…
Cancel
Save