Compare commits

...

2 Commits

Author SHA1 Message Date
Drake 35b13cac0d slight http session cleanup + alternate instance test
ci/woodpecker/push/woodpecker Pipeline failed Details
1 year ago
Drake 357097ab7d make chapters manually inited
1 year ago

@ -4,4 +4,5 @@ export default AO3;
import Chapter from "./src/classes/Chapter.ts";
import Search from "./src/classes/Search.ts";
import Work from "./src/classes/Work.ts";
export { AO3, Chapter, Search, Work };
const VERSION = "v0.2.0";
export { AO3, Chapter, Search, VERSION, Work };

@ -3,73 +3,23 @@ import { ID } from "../types.d.ts";
import { DOMParser } from "https://deno.land/x/deno_dom@v0.1.36-alpha/deno-dom-wasm.ts";
import { HTMLDocument } from "../types.d.ts";
import Search, { SearchParameters } from "./Search.ts";
import {
CookieJar,
wrapFetch,
} from "https://deno.land/x/another_cookiejar@v5.0.1/mod.ts";
import { newSession, Options } from "../utils/http.ts";
export default class AO3 {
session: {
get: (path: string) => Promise<Response>;
post: (
path: string,
payload: unknown,
) => Promise<Response>;
};
session: ReturnType<typeof newSession>;
DOMParser = new DOMParser();
fetch: typeof fetch;
cookieJar: CookieJar;
#headers: Record<string, string>;
/**
* a representation of AO3 in class form
*/
constructor(opts?: {
url?: string;
}) {
this.cookieJar = new CookieJar();
this.fetch = wrapFetch({ cookieJar: this.cookieJar });
this.#headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; rv:108.0) Gecko/20100101 Firefox/108.0",
};
this.session = {
get: async (path: string) => {
const res = await this.fetch(
opts?.url ?? "https://archiveofourown.org/" + path,
{
headers: this.#headers,
},
);
if (res.status > 300) {
console.log(res);
throw new Error("Failed request, probably rate-limited");
}
return res;
},
post: async (
path: string,
// deno-lint-ignore no-explicit-any
payload: any,
headers?: string,
) => {
const res = await this.fetch(
opts?.url ?? "https://archiveofourown.org/" + path,
{
"credentials": "include",
headers: Object.assign(headers ?? {}, this.#headers),
method: "POST",
body: payload,
},
);
if (res.status > 300) {
console.log(res);
throw new Error("Failed request, probably rate-limited");
}
return res;
},
};
constructor(opts?: Options) {
if (opts && !opts.headers) {
opts.headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; rv:108.0) Gecko/20100101 Firefox/108.0",
};
}
this.session = newSession(opts ?? {});
}
/**

@ -4,26 +4,16 @@ export default class Chapter {
#session: {
get: (path: string) => Promise<Response>;
};
isInited = false;
#document!: HTMLDocument;
#DOMParser: DOMParser;
#id: ID;
#workID: ID;
#name!: string;
#html!: string;
#text!: string;
#summary!: string;
#startNote!: string;
#endNote!: string;
earlyName?: Promise<string>;
id!: Promise<ID>;
workID!: Promise<ID>;
name!: Promise<string>;
html!: Promise<string>;
text!: Promise<string>;
summary!: Promise<string>;
startNote!: Promise<string>;
endNote!: Promise<string>;
id: ID;
workID: ID;
name!: string;
html!: string;
text!: string;
summary!: string;
startNote!: string;
endNote!: string;
constructor(
workId: ID,
@ -36,45 +26,15 @@ export default class Chapter {
extraInfo: Record<string, any>,
) {
this.#session = session;
this.#workID = workId;
this.#id = id;
this.workID = workId;
this.id = id;
this.#DOMParser = DOMParser;
this.earlyName = extraInfo.name;
return new Proxy(this, {
get: async (target, prop) => {
if (prop === "earlyName") {
return this.earlyName;
}
if (!this.isInited) {
await target.init();
target.isInited = true;
}
switch (prop) {
case "id":
return target.#id;
case "workID":
return target.#workID;
case "name":
return target.#name;
case "html":
return target.#html;
case "text":
return target.#text;
case "summary":
return target.#summary;
case "startNote":
return target.#startNote;
case "endNote":
return target.#endNote;
}
},
});
this.name = extraInfo.name;
}
async init() {
const res = await this.#session.get(
`/works/${this.#workID}/chapters/${this.#id}?view_adult=true`,
`/works/${this.workID}/chapters/${this.id}?view_adult=true`,
);
this.#document = this.#DOMParser.parseFromString(
await res.text(),
@ -93,7 +53,7 @@ export default class Chapter {
}
populateMetadata() {
this.#name = this.#document.querySelector("h3.title")?.innerText
this.name = this.#document.querySelector("h3.title")?.innerText
.replace(
/Chapter \d+: /,
"",
@ -101,7 +61,7 @@ export default class Chapter {
}
populateSummary() {
this.#summary = this.#document.querySelector("#summary > .userstuff")
this.summary = this.#document.querySelector("#summary > .userstuff")
?.innerText.trim() as string;
}
@ -109,14 +69,14 @@ export default class Chapter {
const notesList = Array.from(
this.#document.querySelectorAll(".notes > .userstuff"),
).map((n) => (n as Element).innerHTML);
this.#startNote = notesList[0]?.trim()?.replace(/<\/{0,1}p>/g, "\n")
this.startNote = notesList[0]?.trim()?.replace(/<\/{0,1}p>/g, "\n")
?.trim();
this.#endNote = notesList[1]?.trim()?.replace(/<\/{0,1}p>/g, "\n")
this.endNote = notesList[1]?.trim()?.replace(/<\/{0,1}p>/g, "\n")
?.trim();
}
async populateText() {
this.#text = "";
this.text = "";
const elements = this.#document.querySelectorAll(
"div.userstuff[role='article'] > p",
@ -125,25 +85,25 @@ export default class Chapter {
for (let i = 0; i < elements.length; i++) {
const element = elements[i] as Element;
this.#text += element.innerText + "\n";
this.text += element.innerText + "\n";
}
try {
this.#text = this.#text.trim();
this.text = this.text.trim();
this.#html = (this.#document.querySelector(
this.html = (this.#document.querySelector(
"div.userstuff[role='article']",
) as Element).innerHTML;
} catch {
//assume single chapter work
const res = await this.#session.get(
`/works/${this.#workID}?view_adult=true`,
`/works/${this.workID}?view_adult=true`,
);
this.#document = this.#DOMParser.parseFromString(
await res.text(),
"text/html",
) as HTMLDocument;
this.#html = (this.#document.querySelector(
this.html = (this.#document.querySelector(
"[role='article'] > div.userstuff",
) as Element).innerHTML;
@ -154,8 +114,8 @@ export default class Chapter {
for (let i = 0; i < elements.length; i++) {
const element = elements[i] as Element;
this.#text += element.innerText + "\n";
this.#html += element.innerHTML;
this.text += element.innerText + "\n";
this.html += element.innerHTML;
}
}
}

@ -75,19 +75,6 @@ export default class Work {
}
populateTags() {
/* this.#document.querySelectorAll("dd.fandom > ul.commas > li").map(
(t) => this.tags.push(t.text),
);
this.#document.querySelectorAll("dd.relationship > ul.commas > li").map(
(t) => this.tags.push(t.text),
);
this.#document.querySelectorAll("dd.character > ul.commas > li").map(
(t) => this.tags.push(t.text),
);
this.#document.querySelectorAll("dd.freeform > ul.commas > li").map(
(t) => this.tags.push(t.text),
); */
const elements = this.#document.querySelectorAll("dd > ul.commas > li");
for (let i = 0; i < elements.length; i++) {

@ -0,0 +1,66 @@
// custom fetch wrappers.. for reasons
import {
CookieJar,
wrapFetch,
} from "https://deno.land/x/another_cookiejar@v5.0.1/mod.ts";
const cookieJar = new CookieJar();
const wrappedFetch = wrapFetch({
cookieJar,
});
const defaultOptions = {
url: "https://archiveofourown.org/",
headers: {
"User-Agent":
`Mozilla/5.0 (Windows NT 10.0; rv:108.0) Gecko/20100101 Firefox/108.0`,
},
};
export interface Options {
url?: string;
headers?: Record<string, string>;
}
const newSession = (opts: Options) => {
opts = Object.assign(defaultOptions, opts);
return {
get: async (path: string) => {
const res = await wrappedFetch(
opts.url + path,
{
headers: opts.headers,
},
);
if (res.status > 300) {
console.log(res);
throw new Error("Failed request, probably rate-limited");
}
return res;
},
post: async (
path: string,
// deno-lint-ignore no-explicit-any
payload: any,
headers?: string,
) => {
const res = await wrappedFetch(
opts.url + path,
{
"credentials": "include",
headers: Object.assign(headers ?? {}, opts.headers),
method: "POST",
body: payload,
},
);
if (res.status > 300) {
console.log(res);
throw new Error("Failed request, probably rate-limited");
}
return res;
},
};
};
export { newSession };

@ -1,40 +1,56 @@
//FIXME: we need to test single-chapter works too (because those seem to be really inconsistent for some reason?)
import AO3 from "../mod.ts";
import { assert } from "https://deno.land/std@0.167.0/testing/asserts.ts";
import {
assert,
AssertionError,
} from "https://deno.land/std@0.167.0/testing/asserts.ts";
export default function test(ao3: AO3) {
Deno.test("chapters", async (test) => {
const work = await ao3.getWork("43251729");
await work.init();
assert(work.chapters.length > 0, "chapters array is not initialized");
await test.step("IDs", async () => {
await test.step("initialization", async () => {
assert(
await work.chapters[0].id === "108714198",
work.chapters.length > 0,
"chapters array is not initialized",
);
try {
await work.chapters[0].init();
await work.chapters[3].init();
} catch {
throw new AssertionError("failed to initialize chapter");
}
});
await test.step("IDs", () => {
assert(
work.chapters[0].id === "108714198",
"incorrect chapter ID",
);
assert(
await work.chapters[0].workID === "43251729",
work.chapters[0].workID === "43251729",
"incorrect work ID",
); //why do we even store the work's ID publicly in a chapter?
});
await test.step("name", async () => {
await test.step("name", () => {
assert(
await work.chapters[0].name === "Welcome to the Studio",
work.chapters[0].name === "Welcome to the Studio",
"incorrect/missing chapter names",
);
});
await test.step("content", async () => {
await test.step("content", () => {
//FIXME: this should probably be tested better
assert(
(await work.chapters[0].text).length > 0,
(work.chapters[0].text).length > 0,
"text/content is completely missing",
);
});
await test.step("notes and summary", async () => {
await test.step("notes and summary", () => {
//FIXME: write a chapter of my fic (lol) that includes a summary and end note for testing
assert(
await work.chapters[3].startNote ===
work.chapters[3].startNote ===
`If you haven't noticed yet, most of these chapters are named after Bendy fansongs
This is definitely because I'm trying to be smart and cool and make funny references, and definitely not because I'm uncreative :)`,

@ -2,9 +2,11 @@ import AO3 from "../src/classes/AO3.ts";
import workTest from "./work.ts";
import chaptersTest from "./chapter.ts";
import searchTest from "./search.ts";
import miscTest from "./misc.ts";
const ao3 = new AO3();
await workTest(ao3);
await chaptersTest(ao3);
await searchTest(ao3);
await miscTest(ao3);

@ -0,0 +1,20 @@
import AO3 from "../mod.ts";
import { assert } from "https://deno.land/std@0.167.0/testing/asserts.ts";
export default function test(ao3: AO3) {
Deno.test("misc", async (test) => {
await test.step("alternate instances", async () => {
const ao3 = new AO3({
url: "https://squidgeworld.org/",
});
const work = await ao3.getWork(34491);
await work.init();
assert(
work.name === "Implementing OTW's Code To Build SquidgeWorld",
"failed to get a work from alternate instance",
);
});
});
}

@ -1,5 +1,4 @@
import AO3 from "../mod.ts";
import type { Search } from "../mod.ts";
import { assert } from "https://deno.land/std@0.167.0/testing/asserts.ts";
export default function test(ao3: AO3) {

Loading…
Cancel
Save