diff --git a/codegen/README.md b/codegen/README.md new file mode 100644 index 00000000..6c645ca3 --- /dev/null +++ b/codegen/README.md @@ -0,0 +1 @@ +codegen using `ast` to generate `sync_api` from `async_api` diff --git a/codegen/__init__.py b/codegen/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/codegen/__main__.py b/codegen/__main__.py new file mode 100644 index 00000000..2c4be615 --- /dev/null +++ b/codegen/__main__.py @@ -0,0 +1,346 @@ +from __future__ import annotations +import re +import argparse +import contextlib +import os +from copy import deepcopy +from typing import Any, TypedDict, cast, Optional, TYPE_CHECKING, Sequence +import ast +from pathlib import Path +import json +import subprocess +import libcst + +if TYPE_CHECKING: + from _typeshed import StrPath + + +PRE_CODEGEN_NAME = "IS_PRE_CODEGEN" +STATICALLY_ASYNC_NAME = "IS_ASYNC" +DYNAMICALLY_ASYNC_NAME = "IS_ASYNC" + +COMMENT_IDENTIFIER = "COMMENT" +PREVIOUS_LINE_COMMENT_IDENTIFIER = "PREV_LINE_COMMENT" + + +class CodegenConfig(TypedDict): + sync_target_directory: str + async_target_directory: str + exclude: list[str] + include_directories: list[str] + + +class AsyncCodegenNodeTransformer(ast.NodeTransformer): + def visit_FunctionDef(self, node: ast.FunctionDef) -> Any: + self.generic_visit(node) + + if node.name.endswith("_prim_sync"): + return None + + return node + + def _is_statically_async_literal(self, node: ast.AST) -> bool: + return isinstance(node, ast.Constant) and node.value == STATICALLY_ASYNC_NAME + + def _is_pre_codegen_literal(self, node: ast.AST) -> bool: + return isinstance(node, ast.Constant) and node.value == PRE_CODEGEN_NAME + + def _match_static_condition(self, test: ast.AST) -> bool | None: + if self._is_statically_async_literal(test): + return True + + if self._is_pre_codegen_literal(test): + return False + + if ( + isinstance(test, ast.UnaryOp) + and isinstance(test.op, ast.Not) + and self._is_statically_async_literal(test.operand) + ): + return False + + if ( + isinstance(test, ast.UnaryOp) + and isinstance(test.op, ast.Not) + and self._is_pre_codegen_literal(test.operand) + ): + return True + + return None + + def visit_If(self, node: ast.If) -> Any: + self.generic_visit(node) + + if (condition_value := self._match_static_condition(node.test)) is not None: + return node.body if condition_value else node.orelse + + return node + + +class SyncCodegenNodeTransformer(ast.NodeTransformer): + def visit_Assign(self, node: ast.Assign) -> Any: + self.generic_visit(node) + + if node.targets: + first_target = node.targets[0] + if isinstance(first_target, ast.Name) and first_target.id == DYNAMICALLY_ASYNC_NAME: + node.value = ast.Constant(value=False, kind=None) + + return node + + def visit_Await(self, node: ast.Await) -> Any: + self.generic_visit(node) + return node.value + + def visit_AsyncFor(self, node: ast.AsyncFor) -> Any: + self.generic_visit(node) + new_node = ast.For(**{field: getattr(node, field) for field in node._fields}) + return ast.copy_location(new_node, node) + + def visit_AsyncWith(self, node: ast.AsyncWith) -> Any: + self.generic_visit(node) + new_node = ast.With(**{field: getattr(node, field) for field in node._fields}) + return ast.copy_location(new_node, node) + + def visit_comprehension(self, node: ast.comprehension) -> Any: + self.generic_visit(node) + node.is_async = 0 + return node + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Any: + self.generic_visit(node) + + # primitive functions are implemented as sync and as async so the async variant can be dropped + if node.name.endswith("_prim"): + return None + + new_node = ast.FunctionDef(**{field: getattr(node, field) for field in node._fields}) + + return ast.copy_location(new_node, node) + + def _is_statically_async_literal(self, node: ast.AST) -> bool: + return isinstance(node, ast.Constant) and node.value == STATICALLY_ASYNC_NAME + + def _is_pre_codegen_literal(self, node: ast.AST) -> bool: + return isinstance(node, ast.Constant) and node.value == PRE_CODEGEN_NAME + + def _match_static_condition(self, test: ast.AST) -> bool | None: + if self._is_statically_async_literal(test): + return False + + if self._is_pre_codegen_literal(test): + return False + + if ( + isinstance(test, ast.UnaryOp) + and isinstance(test.op, ast.Not) + and self._is_statically_async_literal(test.operand) + ): + return True + + if ( + isinstance(test, ast.UnaryOp) + and isinstance(test.op, ast.Not) + and self._is_pre_codegen_literal(test.operand) + ): + return True + + return None + + def visit_If(self, node: ast.If) -> Any: + self.generic_visit(node) + + if (condition_value := self._match_static_condition(node.test)) is not None: + return node.body if condition_value else node.orelse + + return node + + def visit_FunctionDef(self, node: ast.FunctionDef) -> Any: + self.generic_visit(node) + + if node.name.endswith("_prim_sync"): + node.name = node.name.removesuffix("_sync") + + return node + + +class CommentTransformer(libcst.CSTTransformer): + @staticmethod + def _get_comment(node: libcst.CSTNode, target_name: str) -> str | None: + if isinstance(node, libcst.SimpleStatementLine) and len(node.body) == 1: + expr = node.body[0] + if ( + isinstance(expr, libcst.Expr) + and isinstance((call := expr.value), libcst.Call) + and isinstance((func := call.func), libcst.Name) + and func.value == target_name + and len(call.args) == 1 + and isinstance((comment_value := call.args[0].value), libcst.SimpleString) + ): + comment_value_string = comment_value.evaluated_value + if isinstance(comment_value_string, bytes): + return comment_value_string.decode() + return comment_value_string + return None + + @classmethod + def _process_statement_sequence( + cls, sequence: Sequence[libcst.BaseStatement] + ) -> tuple[list[libcst.BaseStatement], list[libcst.EmptyLine], libcst.Comment | None]: + new_body: list[libcst.BaseStatement] = [] + pending_comments: list[libcst.EmptyLine] = [] + header_comment: libcst.Comment | None = None + for statement in sequence: + prev_line_comment_text = cls._get_comment(statement, PREVIOUS_LINE_COMMENT_IDENTIFIER) + if prev_line_comment_text is not None: + comment = libcst.Comment(f"# {prev_line_comment_text}") + if new_body: + prev_statement = new_body[-1] + if isinstance(prev_statement, libcst.SimpleStatementLine): + whitespace = libcst.TrailingWhitespace( + whitespace=libcst.SimpleWhitespace(" "), comment=comment + ) + new_body[-1] = prev_statement.with_changes(trailing_whitespace=whitespace) + else: + pending_comments.append(libcst.EmptyLine(indent=True, comment=comment)) + elif header_comment is None: + header_comment = comment + else: + pending_comments.append(libcst.EmptyLine(indent=True, comment=comment)) + continue + comment_text = cls._get_comment(statement, COMMENT_IDENTIFIER) + if comment_text is not None: + pending_comments.append( + libcst.EmptyLine(indent=True, comment=libcst.Comment(f"# {comment_text}")) + ) + continue + if ( + pending_comments + and (leading_lines := getattr(statement, "leading_lines", None)) is not None + ): + statement = statement.with_changes( + leading_lines=(*leading_lines, *pending_comments) + ) + pending_comments = [] + new_body.append(statement) + return (new_body, pending_comments, header_comment) + + def leave_Module( + self, original_node: libcst.Module, updated_node: libcst.Module + ) -> libcst.Module: + new_body, pending_comments, header_comment = self._process_statement_sequence( + updated_node.body + ) + if header_comment is not None: + pending_comments.insert(0, libcst.EmptyLine(indent=True, comment=header_comment)) + new_footer = (*updated_node.footer, *pending_comments) + return updated_node.with_changes(body=new_body, footer=new_footer) + + def leave_IndentedBlock( + self, original_node: libcst.IndentedBlock, updated_node: libcst.IndentedBlock + ) -> libcst.IndentedBlock: + new_body, pending_comments, header_comment = self._process_statement_sequence( + updated_node.body + ) + new_footer = (*updated_node.footer, *pending_comments) + if not new_body: + new_body = [libcst.SimpleStatementLine([libcst.Pass()])] + + changes: dict[str, Any] = {"body": new_body, "footer": new_footer} + + if header_comment: + changes["header"] = libcst.TrailingWhitespace( + whitespace=libcst.SimpleWhitespace(" "), comment=header_comment + ) + + return updated_node.with_changes(**changes) + + +def codegen_for_ast(ast: ast.AST) -> tuple[ast.AST, ast.AST]: + ast_2 = deepcopy(ast) + return ( + SyncCodegenNodeTransformer().generic_visit(ast), + AsyncCodegenNodeTransformer().generic_visit(ast_2), + ) + + +def add_comments(code: str) -> str: + cst = libcst.parse_module(code) + transformer = CommentTransformer() + return cst.visit(transformer).code + + +def codegen_for_file(file: Path) -> tuple[ast.AST, ast.AST]: + code = file.read_text() + return codegen_for_ast(ast.parse(code)) + + +def codegen_for_whole_directory(directory: "StrPath"): + directory = Path(directory).resolve() + items = {path.name: path for path in directory.iterdir()} + codegen_config: CodegenConfig + try: + codegen_config = cast( + "CodegenConfig", json.loads(items.pop("codegen_config.json").read_text()) + ) + except KeyError: + codegen_config = CodegenConfig( + sync_target_directory=str( + directory.with_stem(f"{directory.stem}_sync"), + ), + async_target_directory=str( + directory.with_stem(f"{directory.stem}_async"), + ), + exclude=[], + include_directories=[], + ) + sync_target_directory = directory / codegen_config["sync_target_directory"] + async_target_directory = directory / codegen_config["async_target_directory"] + sync_target_directory.mkdir(parents=True, exist_ok=True) + async_target_directory.mkdir(parents=True, exist_ok=True) + exclusions = {(directory / exclusion).resolve() for exclusion in codegen_config["exclude"]} + for path in items.values(): + path = path.resolve() + if path.suffix.lower() != ".py": + continue + if not path.is_file(): + continue + if path in exclusions: + continue + (sync_ast, async_ast) = codegen_for_file(path) + (sync_code, async_code) = ( + add_comments(ast.unparse(sync_ast)), + add_comments(ast.unparse(async_ast)), + ) + (sync_target_directory / path.name).write_text(sync_code) + (async_target_directory / path.name).write_text(async_code) + subprocess.run( + [ + "python", + "-m", + "ruff", + "format", + str(sync_target_directory.resolve()), + str(async_target_directory.resolve()), + ], + capture_output=True, + text=True, + ) + for included_dir in codegen_config["include_directories"]: + codegen_for_whole_directory(directory / included_dir) + + +class CodegenArgumentNamespace(argparse.Namespace): + targets: list[Path] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("targets", nargs="*", type=Path) + parsed = parser.parse_args(namespace=CodegenArgumentNamespace()) + for target in parsed.targets: + codegen_for_whole_directory(target) + + +if __name__ == "__main__": + main() diff --git a/codegen/features.md b/codegen/features.md new file mode 100644 index 00000000..090e6875 --- /dev/null +++ b/codegen/features.md @@ -0,0 +1,120 @@ +# Features + +The features supported by the codegen module are as follows: + +## Automatic formatting + +Codegen automatically formats result files using ruff. This cannot currently be disabled. + +## Dynamic checking + +You can declare a variable `IS_ASYNC` like so (recommended to be in global scope): + +```python +IS_ASYNC = True +``` + +It will be `True` in the async code and `False` in the sync code. This can be used to check dynamically where you currently are. + +More specifically, any assignment of `IS_ASYNC` will be left as is in the async code and will be changed to assign `False` in the sync code. + +## Static checking + +### `"IS_ASYNC"` + +If you use exactly `"IS_ASYNC"` as a condition (and nothing else) in an if, if else or if elif else statement, only paths with it being true will be included in the async code and only paths with it being false will be included in the sync code. + +Example: + +```python +if "IS_ASYNC": + async_implementation() +else: + sync_implementation() +``` + +Codegen will turn this into (respectively async and sync): + +```python +async_implementation() +``` +and +```python +sync_implementation() +``` + +### `"IS_PRE_CODEGEN"` + +If you use exactly `"IS_PRE_CODEGEN"` as a condition (and nothing else) in an if, if else or if elif else statement, only paths with it being false will be included in the async and sync code. The purpose of this feature is to allow you to satisfy static code analysis without bloating the resulting code. + +Example: + +```python +if "IS_PRE_CODEGEN": + import requests + import aiohttp +else: + if "IS_ASYNC": + import aiohttp + else: + import requests +``` + +Codegen will turn this into (respectively async and sync): + +```python +import aiohttp +``` +and +```python +import requests +``` + +## Comment inclusion + +Because the first pass of the codegen module will create an ast from the code, all comments in the original code will be removed. If you need comments, exempli gratia for explicitly ignoring type errors, you will need one of these. + +### `COMMENT` + +If you call a function `COMMENT` with exactly one argument that is a simple literal string, it will expand into a line comment. + +Example: + +```python +if "IS_PRE_CODEGEN": + from typing import LiteralString + + def COMMENT(msg: LiteralString) -> None: ... + +COMMENT("Hello there!") +``` + +Codegen will turn this into: + +```python +# Hello there! +``` + +### `PREV_LINE_COMMENT` + +`COMMENT` will likely not satisfy your needs. If you call a function `PREV_LINE_COMMENT` with exactly one argument that is a simple literal string, it will expand into a line comment that is located at the end of the previous statement. + +Example: + +```python +if "IS_PRE_CODEGEN": + from typing import LiteralString + + def PREV_LINE_COMMENT(msg: LiteralString) -> None: ... + +def func(): + PREV_LINE_COMMENT("typing: ignore") + pass +``` + +Codegen will turn this into: + +```python +def func(): # typing: ignore + pass +``` diff --git a/codegen/pyproject.toml b/codegen/pyproject.toml new file mode 100644 index 00000000..10763bf8 --- /dev/null +++ b/codegen/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "codegen" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.12.12" +dependencies = ["ruff", "libcst"] diff --git a/codegen/test_codegen/async_out/test.py b/codegen/test_codegen/async_out/test.py new file mode 100644 index 00000000..48aceb89 --- /dev/null +++ b/codegen/test_codegen/async_out/test.py @@ -0,0 +1,58 @@ +from typing import Iterable, TypeVar, ParamSpec, Generic, Any, TYPE_CHECKING, Optional, cast +from collections.abc import Callable +import time + +IS_ASYNC = True +if IS_ASYNC: + from collections.abc import Awaitable + import asyncio +else: + import threading + from dataclasses import dataclass +P = ParamSpec("P") +O = TypeVar("O") + + +def create_task(function: Callable[P, O], *args: P.args, **kwargs: P.kwargs) -> O: + return function(*args, **kwargs) + + +async def sleep_prim(delay: int | float): + await asyncio.sleep(delay) + + +T = TypeVar("T") + + +async def gather_prim(*tasks: Awaitable[T]) -> list[T]: + return await asyncio.gather(*tasks) + + +async def fetch_user_data(user_id: int, delay: int) -> dict: + print(f"[{time.strftime('%X')}] Task {user_id}: Starting request (takes {delay}s)...") + await sleep_prim(delay) + print(f"[{time.strftime('%X')}] Task {user_id}: Finished request!") + return {"user_id": user_id, "status": "success"} + + +async def main(): + start_time = time.perf_counter() + print("--- Fetching data concurrently ---") + coroutines = [ + create_task(fetch_user_data, user_id=1, delay=2), + create_task(fetch_user_data, user_id=2, delay=3), + create_task(fetch_user_data, user_id=3, delay=1), + ] + results = await gather_prim(*coroutines) + end_time = time.perf_counter() + total_time = end_time - start_time + print("\n--- All tasks complete ---") + print(f"Total time taken: {total_time:.2f} seconds") + print("Results:", results) + + +if __name__ == "__main__": + if IS_ASYNC: + asyncio.run(main()) + else: + main() diff --git a/codegen/test_codegen/codegen_config.json b/codegen/test_codegen/codegen_config.json new file mode 100644 index 00000000..6e8371cc --- /dev/null +++ b/codegen/test_codegen/codegen_config.json @@ -0,0 +1,6 @@ +{ + "sync_target_directory": "./sync_out", + "async_target_directory": "./async_out", + "exclude": [], + "include_directories": [] +} \ No newline at end of file diff --git a/codegen/test_codegen/sync_out/test.py b/codegen/test_codegen/sync_out/test.py new file mode 100644 index 00000000..72e951c1 --- /dev/null +++ b/codegen/test_codegen/sync_out/test.py @@ -0,0 +1,76 @@ +from typing import Iterable, TypeVar, ParamSpec, Generic, Any, TYPE_CHECKING, Optional, cast +from collections.abc import Callable +import time + +IS_ASYNC = False +if IS_ASYNC: + from collections.abc import Awaitable + import asyncio +else: + import threading + from dataclasses import dataclass +P = ParamSpec("P") +O = TypeVar("O") + + +@dataclass +class Task(Generic[O]): + out: Optional[O] + thread: threading.Thread + + +def create_task(function: Callable[P, O], *args: P.args, **kwargs: P.kwargs) -> Task[O]: + task: Task[O] = Task(None, cast(threading.Thread, None)) + + def wrapper(*args, **kwargs): + task.out = function(*args, **kwargs) + + task.thread = threading.Thread(target=wrapper, args=args, kwargs=kwargs) + return task + + +def sleep_prim(delay: int | float): + time.sleep(delay) + + +T = TypeVar("T") + + +def gather_prim(*tasks: Task[T]) -> list[T]: + values: list[T] = [] + for task in tasks: + task.thread.start() + for task in tasks: + task.thread.join() + values.append(cast(T, task.out)) + return values + + +def fetch_user_data(user_id: int, delay: int) -> dict: + print(f"[{time.strftime('%X')}] Task {user_id}: Starting request (takes {delay}s)...") + sleep_prim(delay) + print(f"[{time.strftime('%X')}] Task {user_id}: Finished request!") + return {"user_id": user_id, "status": "success"} + + +def main(): + start_time = time.perf_counter() + print("--- Fetching data concurrently ---") + coroutines = [ + create_task(fetch_user_data, user_id=1, delay=2), + create_task(fetch_user_data, user_id=2, delay=3), + create_task(fetch_user_data, user_id=3, delay=1), + ] + results = gather_prim(*coroutines) + end_time = time.perf_counter() + total_time = end_time - start_time + print("\n--- All tasks complete ---") + print(f"Total time taken: {total_time:.2f} seconds") + print("Results:", results) + + +if __name__ == "__main__": + if IS_ASYNC: + asyncio.run(main()) + else: + main() diff --git a/codegen/test_codegen/test.py b/codegen/test_codegen/test.py new file mode 100644 index 00000000..81960370 --- /dev/null +++ b/codegen/test_codegen/test.py @@ -0,0 +1,110 @@ +from typing import Iterable, TypeVar, ParamSpec, Generic, Any, TYPE_CHECKING, Optional, cast +from collections.abc import Callable + +import time + +IS_ASYNC = True +if IS_ASYNC: + from collections.abc import Awaitable + import asyncio +else: + import threading + from dataclasses import dataclass + + +P = ParamSpec("P") +O = TypeVar("O") + +if "IS_PRE_CODEGEN": + if TYPE_CHECKING: + import threading + from dataclasses import dataclass + + @dataclass + class Task(Generic[O]): + out: Optional[O] + thread: threading.Thread + + +if "IS_ASYNC": + + def create_task(function: Callable[P, O], *args: P.args, **kwargs: P.kwargs) -> O: + return function(*args, **kwargs) +else: + + @dataclass + class Task(Generic[O]): # type: ignore[no-redef] + out: Optional[O] + thread: threading.Thread + + def create_task(function: Callable[P, O], *args: P.args, **kwargs: P.kwargs) -> Task[O]: # type: ignore[misc] + task: Task[O] = Task(None, cast(threading.Thread, None)) # type: ignore[arg-type] + + def wrapper(*args, **kwargs): + task.out = function(*args, **kwargs) + + task.thread = threading.Thread(target=wrapper, args=args, kwargs=kwargs) + return task + + +def sleep_prim_sync(delay: int | float): + time.sleep(delay) + + +async def sleep_prim(delay: int | float): + await asyncio.sleep(delay) + + +T = TypeVar("T") + + +def gather_prim_sync(*tasks: Task[T]) -> list[T]: + values: list[T] = [] + for task in tasks: + task.thread.start() + + for task in tasks: + task.thread.join() + values.append(cast(T, task.out)) + + return values + + +async def gather_prim(*tasks: Awaitable[T]) -> list[T]: + return await asyncio.gather(*tasks) + + +async def fetch_user_data(user_id: int, delay: int) -> dict: + print(f"[{time.strftime('%X')}] Task {user_id}: Starting request (takes {delay}s)...") + + await sleep_prim(delay) + + print(f"[{time.strftime('%X')}] Task {user_id}: Finished request!") + return {"user_id": user_id, "status": "success"} + + +async def main(): + start_time = time.perf_counter() + print("--- Fetching data concurrently ---") + + coroutines = [ + create_task(fetch_user_data, user_id=1, delay=2), + create_task(fetch_user_data, user_id=2, delay=3), + create_task(fetch_user_data, user_id=3, delay=1), + ] + + results = await gather_prim(*coroutines) + + end_time = time.perf_counter() + total_time = end_time - start_time + + print("\n--- All tasks complete ---") + print(f"Total time taken: {total_time:.2f} seconds") + print("Results:", results) + + +if __name__ == "__main__": + if IS_ASYNC: + asyncio.run(main()) + else: + main() diff --git a/pyproject.toml b/pyproject.toml index b3a8d4fb..c2135ace 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,6 +97,11 @@ max-complexity = 10 [tool.uv] config-settings = { editable_mode = "compat" } +[tool.uv.workspace] +members = [ + "codegen", +] + [tool.setuptools.packages.find] where = ["."] include = ["scratchattach*"] diff --git a/scratchattach/_core/__init__.py b/scratchattach/_core/__init__.py new file mode 100644 index 00000000..f0dc916a --- /dev/null +++ b/scratchattach/_core/__init__.py @@ -0,0 +1,4 @@ +""" +Async implementations for blocking operations that are not to be used themselves. +The implementations are read by the global codegen module and async and sync variations are generated. +""" \ No newline at end of file diff --git a/scratchattach/_core/codegen_config.json b/scratchattach/_core/codegen_config.json new file mode 100644 index 00000000..c0a31994 --- /dev/null +++ b/scratchattach/_core/codegen_config.json @@ -0,0 +1,8 @@ +{ + "sync_target_directory": "../sync_api", + "async_target_directory": "../async_api", + "exclude": [ + "./__init__.py" + ], + "include_directories": ["./primitives", "./site"] +} \ No newline at end of file diff --git a/scratchattach/_core/primitives/__init__.py b/scratchattach/_core/primitives/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scratchattach/_core/primitives/codegen_config.json b/scratchattach/_core/primitives/codegen_config.json new file mode 100644 index 00000000..673f7408 --- /dev/null +++ b/scratchattach/_core/primitives/codegen_config.json @@ -0,0 +1,8 @@ +{ + "sync_target_directory": "../../sync_api/primitives", + "async_target_directory": "../../async_api/primitives", + "exclude": [ + "./__init__.py" + ], + "include_directories": [] +} \ No newline at end of file diff --git a/scratchattach/_core/primitives/http.py b/scratchattach/_core/primitives/http.py new file mode 100644 index 00000000..b45571ac --- /dev/null +++ b/scratchattach/_core/primitives/http.py @@ -0,0 +1,550 @@ +from __future__ import annotations +from types import TracebackType +from collections.abc import Iterable, Mapping +from typing import Optional, Self, cast, Any, Sequence, SupportsInt, BinaryIO, TYPE_CHECKING + +if TYPE_CHECKING: + from _typeshed import SupportsKeysAndGetItem + +from scratchattach._shared import http as shared_http + +if "IS_PRE_CODEGEN": + import contextlib + from typing import LiteralString + import aiohttp + import requests + from requests import cookies as requests_cookies + + def PREV_LINE_COMMENT(msg: LiteralString) -> None: ... +else: + if "IS_ASYNC": + import contextlib + import aiohttp + else: + import requests + from requests import cookies as requests_cookies + + +HTTPOptions = shared_http.HTTPOptions + + +if "IS_ASYNC": + + class _HTTPResponse: + _async_response: aiohttp.ClientResponse + if "IS_PRE_CODEGEN": + _sync_response: requests.Response + + async def text(self) -> str: + return await self._async_response.text() + + async def content(self) -> bytes: + return await self._async_response.content.read() + + async def json(self) -> Any: + return await self._async_response.json() + + @property + def headers(self) -> Mapping[str, str]: + """ + Headers are case-insensitive. + """ + return self._async_response.headers + + def get_all_headers_for_key(self, key: str) -> list[str]: + return self._async_response.headers.getall(key) + + @property + def status_code(self) -> int: + return self._async_response.status + + class _WrappedHTTPResponse: + _aiohttp_response_context_manager: aiohttp.client._BaseRequestContextManager[ + aiohttp.ClientResponse + ] + + def __init__( + self, + aiohttp_response_context_manager: aiohttp.client._BaseRequestContextManager[ + aiohttp.ClientResponse + ], + ): + self._aiohttp_response_context_manager = aiohttp_response_context_manager + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + # __exit__ should exist in pair with __enter__ but never executed + pass # pragma: no cover + + async def __aenter__(self) -> _HTTPResponse: + http_response = _HTTPResponse() + http_response._async_response = ( + await self._aiohttp_response_context_manager.__aenter__() + ) + return http_response + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ): + await self._aiohttp_response_context_manager.__aexit__(exc_type, exc, tb) + + class _HTTPSession: + def add_cookie(self, key: str, value: str): + self._cookies[key] = value + + def get_cookie(self, key: str) -> Optional[str]: + return self._cookies.get(key) + + def remove_cookie(self, key: str): + del self._cookies[key] + + def clear_cookies(self): + self._cookies.clear() + + def update_cookies(self, new: "SupportsKeysAndGetItem[str, str]"): + self._cookies.update(new) + + def list_cookies(self) -> Iterable[tuple[str, str]]: + return self._cookies.items() + + def add_header(self, key: str, value: str): + self._headers[key] = value + + def get_header(self, key: str) -> Optional[str]: + return self._headers.get(key) + + def remove_header(self, key: str): + del self._headers[key] + + def clear_headers(self): + self._headers.clear() + + def update_headers(self, new: "SupportsKeysAndGetItem[str, str]"): + self._headers.update(new) + + def list_headers(self) -> Iterable[tuple[str, str]]: + return self._headers.items() + + _cookies: dict[str, str] + _headers: dict[str, str] + + _http_session: aiohttp.ClientSession + + def __init__(self): + self._cookies = {} + self._headers = {} + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + # __exit__ should exist in pair with __enter__ but never executed + pass # pragma: no cover + + async def __aenter__(self) -> Self: + self._http_session = aiohttp.ClientSession(cookie_jar=aiohttp.DummyCookieJar()) + await self._http_session.__aenter__() + return self + + async def __aexit__( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ): + await self._http_session.__aexit__(exc_type, exc_val, exc_tb) + + @staticmethod + def _get_params_kwarg( + params: Any, + ) -> ( + None + | str + | Mapping[str, Sequence[str | SupportsInt] | SupportsInt] + | Sequence[tuple[str, Sequence[str | SupportsInt] | SupportsInt]] + ): + if isinstance(params, shared_http.SupportsItems): + params = params.items() + if not isinstance(params, str) and isinstance(params, Iterable): + new_params: Any = [] + for key, value in cast( + Iterable[tuple[str, Iterable[str | SupportsInt] | SupportsInt | None]], params + ): + if not isinstance(value, str) and isinstance(value, Iterable): + for item in value: + new_params.append((key, cast(str | SupportsInt, item))) + elif value is None: + pass + else: + new_params.append((key, cast(str | SupportsInt, value))) + params = new_params + return params + + @staticmethod + def _get_data_and_files_kwargs( + data: Iterable[tuple[str, Any]] | shared_http.SupportsItems[str, Any] | None, + files: Iterable[tuple[str, BinaryIO | bytes]] + | shared_http.SupportsItems[str, BinaryIO | bytes] + | None, + ) -> aiohttp.FormData: + form_data = aiohttp.FormData() + if data is not None: + if isinstance(data, shared_http.SupportsItems): + data = cast(Any, data.items()) + for key, value in cast(Iterable[tuple[str, Any]], data): + form_data.add_field(key, str(value)) + if files is not None: + if isinstance(files, shared_http.SupportsItems): + files = cast(Any, files.items()) + for key, file_data in cast(Iterable[tuple[str, BinaryIO | bytes]], files): + form_data.add_field(key, file_data, filename=key) + return form_data + + @staticmethod + def _get_headers_kwarg( + default_headers: dict[str, str], + headers: Iterable[tuple[str, str]] | shared_http.SupportsItems[str, str] | None, + ) -> dict[str, str]: + if headers is None: + return default_headers + if isinstance(headers, shared_http.SupportsItems): + headers = cast(Iterable[tuple[str, str]], headers.items()) + return default_headers | dict(headers) + + @staticmethod + def _get_cookies_kwarg( + default_cookies: dict[str, str], + cookies: Iterable[tuple[str, str]] | shared_http.SupportsItems[str, str] | None, + ) -> dict[str, str]: + if cookies is None: + return default_cookies + if isinstance(cookies, shared_http.SupportsItems): + cookies = cast(Iterable[tuple[str, str]], cookies.items()) + return default_cookies | dict(cookies) + + def _get_kwargs(self, options: HTTPOptions) -> aiohttp.client._RequestOptions: # noqa: C901 + PREV_LINE_COMMENT("noqa: C901") + kwargs: aiohttp.client._RequestOptions = {} + if options.params is not None: + kwargs["params"] = self._get_params_kwarg(options.params) + if options.content is not None and options.data is not None: + raise ValueError('Cannot specify both "content" and "data"') + if options.content is not None and options.files is not None: + raise ValueError('Cannot specify both "content" and "files"') + if options.content is not None: + kwargs["data"] = options.content + if options.data is not None or options.files is not None: + kwargs["data"] = self._get_data_and_files_kwargs(options.data, options.files) + kwargs["cookies"] = self._get_cookies_kwarg( + {} if options.disregard_default_cookies else self._cookies, options.cookies + ) + kwargs["headers"] = self._get_headers_kwarg( + {} if options.disregard_default_headers else self._headers, options.headers + ) + if options.json is not shared_http._JsonEmptySentinel and ( + options.content is not None or options.data is not None or options.files is not None + ): + raise ValueError('Cannot specify "json" alongside "content", "data", or "files"') + if options.json is not shared_http._JsonEmptySentinel: + kwargs["json"] = options.json + if options.timeout: + kwargs["timeout"] = aiohttp.ClientTimeout( + total=options.timeout, sock_connect=min(30, options.timeout) + ) + return kwargs + + def get(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs( + options if options is not None else shared_http._EMPTY_OPTIONS + ) + return _WrappedHTTPResponse(self._http_session.get(url, **kwargs)) + + def post(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs( + options if options is not None else shared_http._EMPTY_OPTIONS + ) + return _WrappedHTTPResponse(self._http_session.post(url, **kwargs)) + + def put(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs( + options if options is not None else shared_http._EMPTY_OPTIONS + ) + return _WrappedHTTPResponse(self._http_session.put(url, **kwargs)) + + def delete(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs( + options if options is not None else shared_http._EMPTY_OPTIONS + ) + return _WrappedHTTPResponse(self._http_session.delete(url, **kwargs)) + + def request( + self, method: shared_http.HTTPMethod, url: str, options: HTTPOptions | None = None + ) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs( + options if options is not None else shared_http._EMPTY_OPTIONS + ) + return _WrappedHTTPResponse(self._http_session.request(method.name, url, **kwargs)) +else: + + class _HTTPResponse: # type: ignore[no-redef] + _sync_response: requests.Response + + def text(self) -> str: + return self._sync_response.text + + def content(self) -> bytes: + return self._sync_response.content + + def json(self) -> Any: + return self._sync_response.json() + + @property + def headers(self) -> Mapping[str, str]: + """ + Headers are case-insensitive. + """ + return self._sync_response.headers + + def get_all_headers_for_key(self, key: str) -> list[str]: + return self._sync_response.raw.headers.getlist(key) + + @property + def status_code(self) -> int: + return self._sync_response.status_code + + class _WrappedHTTPResponse: # type: ignore[no-redef] + _response: requests.Response + + def __init__(self, response: requests.Response): + self._response = response + + def __enter__(self) -> _HTTPResponse: + response = _HTTPResponse() + response._sync_response = self._response + return response + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self._response.close() + + class DummyCookieJar(requests_cookies.RequestsCookieJar): + def set_cookie(self, *args, **kwargs): + pass + + def update(self, *args, **kwargs): + pass + + def copy(self): + return DummyCookieJar() + + def __setitem__(self, name, value): + pass + + def set(self, *args, **kwargs): + pass + + class _HTTPSession: # type: ignore[no-redef] + def add_cookie(self, key: str, value: str): + self._cookies[key] = value + + def get_cookie(self, key: str) -> Optional[str]: + return self._cookies.get(key) + + def remove_cookie(self, key: str): + del self._cookies[key] + + def clear_cookies(self): + self._cookies.clear() + + def update_cookies(self, new: "SupportsKeysAndGetItem[str, str]"): + self._cookies.update(new) + + def list_cookies(self) -> Iterable[tuple[str, str]]: + return self._cookies.items() + + def add_header(self, key: str, value: str): + self._headers[key] = value + + def get_header(self, key: str) -> Optional[str]: + return self._headers.get(key) + + def remove_header(self, key: str): + del self._headers[key] + + def clear_headers(self): + self._headers.clear() + + def update_headers(self, new: "SupportsKeysAndGetItem[str, str]"): + self._headers.update(new) + + def list_headers(self) -> Iterable[tuple[str, str]]: + return self._headers.items() + + _cookies: dict[str, str] + _headers: dict[str, str] + + _http_session: requests.Session + + def __init__(self): + self._cookies = {} + self._headers = {} + self._http_session = requests.Session() + self._http_session.cookies = DummyCookieJar() + + def __enter__(self) -> Self: + return self + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + pass + + @staticmethod + def _get_params_kwarg(params: Any) -> Any: + if isinstance(params, shared_http.SupportsItems): + params = params.items() + if not isinstance(params, str) and isinstance(params, Iterable): + new_params: Any = [] + for key, value in cast( + Iterable[tuple[str, Iterable[str | SupportsInt] | SupportsInt | None]], params + ): + if not isinstance(value, str) and isinstance(value, Iterable): + for item in value: + new_params.append((key, cast(str | SupportsInt, item))) + elif value is None: + pass + else: + new_params.append((key, cast(str | SupportsInt, value))) + params = new_params + return params + + @staticmethod + def _get_data_and_files_kwargs( + data: Iterable[tuple[str, Any]] | shared_http.SupportsItems[str, Any] | None, + files: Iterable[tuple[str, BinaryIO | bytes]] + | shared_http.SupportsItems[str, BinaryIO | bytes] + | None, + ) -> tuple[list[tuple[str, Any]] | None, list[tuple[str, BinaryIO | bytes]] | None]: + processed_data = None + processed_files = None + if data is not None: + if isinstance(data, shared_http.SupportsItems): + data = cast(Any, data.items()) + processed_data = list(cast(Iterable[tuple[str, Any]], data)) + if files is not None: + if isinstance(files, shared_http.SupportsItems): + files = cast(Any, files.items()) + processed_files = list(cast(Iterable[tuple[str, BinaryIO | bytes]], files)) + return processed_data, processed_files + + @staticmethod + def _get_headers_kwarg( + default_headers: dict[str, str], + headers: Iterable[tuple[str, str]] | shared_http.SupportsItems[str, str] | None, + ) -> dict[str, str]: + if headers is None: + return default_headers + if isinstance(headers, shared_http.SupportsItems): + headers = cast(Iterable[tuple[str, str]], headers.items()) + return default_headers | dict(headers) + + @staticmethod + def _get_cookies_kwarg( + default_cookies: dict[str, str], + cookies: Iterable[tuple[str, str]] | shared_http.SupportsItems[str, str] | None, + ) -> dict[str, str]: + if cookies is None: + return default_cookies + if isinstance(cookies, shared_http.SupportsItems): + cookies = cast(Iterable[tuple[str, str]], cookies.items()) + return default_cookies | dict(cookies) + + def _get_kwargs(self, options: HTTPOptions) -> dict[str, Any]: # noqa: C901 + PREV_LINE_COMMENT("noqa: C901") + kwargs: dict[str, Any] = {} + + if options.params is not None: + kwargs["params"] = self._get_params_kwarg(options.params) + if options.content is not None and options.data is not None: + raise ValueError('Cannot specify both "content" and "data"') + if options.content is not None and options.files is not None: + raise ValueError('Cannot specify both "content" and "files"') + if options.content is not None: + kwargs["data"] = options.content + if options.data is not None or options.files is not None: + processed_data, processed_files = self._get_data_and_files_kwargs( + options.data, options.files + ) + if processed_data is not None: + kwargs["data"] = processed_data + if processed_files is not None: + kwargs["files"] = processed_files + kwargs["cookies"] = self._get_cookies_kwarg( + {} if options.disregard_default_cookies else self._cookies, options.cookies + ) + kwargs["headers"] = self._get_headers_kwarg( + {} if options.disregard_default_headers else self._headers, options.headers + ) + if options.json is not shared_http._JsonEmptySentinel and ( + options.content is not None or options.data is not None or options.files is not None + ): + raise ValueError('Cannot specify "json" alongside "content", "data", or "files"') + if options.json is not shared_http._JsonEmptySentinel: + kwargs["json"] = options.json + if options.timeout: + kwargs["timeout"] = options.timeout + return kwargs + + def get(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs( + options if options is not None else shared_http._EMPTY_OPTIONS + ) + return _WrappedHTTPResponse(self._http_session.get(url, **kwargs)) # type: ignore[arg-type] + + def post(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs( + options if options is not None else shared_http._EMPTY_OPTIONS + ) + return _WrappedHTTPResponse(self._http_session.post(url, **kwargs)) # type: ignore[arg-type] + + def put(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs( + options if options is not None else shared_http._EMPTY_OPTIONS + ) + return _WrappedHTTPResponse(self._http_session.put(url, **kwargs)) # type: ignore[arg-type] + + def delete(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs( + options if options is not None else shared_http._EMPTY_OPTIONS + ) + return _WrappedHTTPResponse(self._http_session.delete(url, **kwargs)) # type: ignore[arg-type] + + def request( + self, method: shared_http.HTTPMethod, url: str, options: HTTPOptions | None = None + ) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs( + options if options is not None else shared_http._EMPTY_OPTIONS + ) + return _WrappedHTTPResponse(self._http_session.request(method.name, url, **kwargs)) # type: ignore[arg-type] diff --git a/scratchattach/_core/primitives/utils.py b/scratchattach/_core/primitives/utils.py new file mode 100644 index 00000000..51c5959d --- /dev/null +++ b/scratchattach/_core/primitives/utils.py @@ -0,0 +1,315 @@ +from __future__ import annotations +from collections.abc import Callable +from typing import Union, ParamSpec, TypeVar, Generic, Any, cast, Optional, overload, Literal +import time + +if "IS_PRE_CODEGEN": + CTYPES_PRESENT = True + import ctypes + import threading + import concurrent.futures + import asyncio + from collections.abc import Awaitable, Coroutine +else: + if "IS_ASYNC": + CTYPES_PRESENT = False + import asyncio + from collections.abc import Awaitable, Coroutine + else: + try: + import ctypes + + CTYPES_PRESENT = True + except Exception: + CTYPES_PRESENT = False + import threading + import concurrent.futures + + +def sleep_prim_sync(delay: Union[int, float]): + time.sleep(delay) + + +async def sleep_prim(delay: Union[int, float]): + await asyncio.sleep(delay) + + +P = ParamSpec("P") +O = TypeVar("O", covariant=True) + + +class Task(Generic[P, O]): + function: Callable[P, O] + args: Any + kwargs: Any + available: bool + + +class LaunchedTask(Generic[P, O]): + task: Task[P, O] + if "IS_PRE_CODEGEN": + _out: O + _task: asyncio.Task[Any] + _thread: threading.Thread + else: + if "IS_ASYNC": + _task: asyncio.Task[Any] # type: ignore[no-redef] + else: + _out: O # type: ignore[no-redef] + _thread: threading.Thread # type: ignore[no-redef] + + +def create_task(function: Callable[P, O], *args: P.args, **kwargs: P.kwargs) -> Task[P, O]: + task: Task[P, O] = Task() + task.function = function + task.args = args + task.kwargs = kwargs + task.available = True + return task + + +def gather_concurrently_prim_sync(*tasks: Task[Any, O]) -> list[O]: + with concurrent.futures.ThreadPoolExecutor() as executor: + return [cast(O, i) for i in executor.map(lambda x: x.function(*x.args, **x.kwargs), tasks)] + + +async def gather_concurrently_prim(*tasks: Task[Any, Awaitable[O]]) -> list[O]: + for task in tasks: + if not task.available: + raise ValueError("Task is already used.") + task.available = False + return await asyncio.gather(*(task.function(*task.args, **task.kwargs) for task in tasks)) + + +def launch_concurrently_prim_sync(task: Task[P, O]) -> LaunchedTask[P, O]: + launched_task: LaunchedTask[P, O] = LaunchedTask() + + def wrap_function(): + launched_task._out = task.function(*task.args, **task.kwargs) + + thread = threading.Thread(target=wrap_function) + thread.start() + launched_task.task = task + launched_task._thread = thread + return launched_task + + +A = TypeVar("A") +B = TypeVar("B") + + +async def launch_concurrently_prim( + task: Task[P, Coroutine[A, B, O]], +) -> LaunchedTask[P, Coroutine[A, B, O]]: + _task = asyncio.create_task(task.function(*task.args, **task.kwargs)) + launched_task: LaunchedTask[P, Coroutine[A, B, O]] = LaunchedTask() + launched_task.task = task + launched_task._task = _task + return launched_task + + +@overload +def join_launched_task_prim_sync(task: LaunchedTask[P, O]) -> O: + pass + + +@overload +def join_launched_task_prim_sync( + task: LaunchedTask[P, O], timeout: Union[float, int] +) -> Optional[O]: + pass + + +def join_launched_task_prim_sync( + task: LaunchedTask[P, O], timeout: Optional[Union[float, int]] = None +) -> Optional[O]: + task._thread.join(timeout) + if task._thread.is_alive(): + return None + return task._out + + +@overload +async def join_launched_task_prim(task: LaunchedTask[P, Coroutine[Any, Any, O]]) -> O: + pass + + +@overload +async def join_launched_task_prim( + task: LaunchedTask[P, Coroutine[Any, Any, O]], timeout: Union[float, int] +) -> Optional[O]: + pass + + +async def join_launched_task_prim( + task: LaunchedTask[P, Coroutine[Any, Any, O]], timeout: Optional[Union[float, int]] = None +) -> Optional[O]: + try: + return await asyncio.wait_for(asyncio.shield(task._task), timeout) + except TimeoutError: + return None + + +if "IS_PRE_CODEGEN": + + def _raise_in_thread(thread: threading.Thread, exc_type: type[BaseException]) -> None: ... + + +if not "IS_ASYNC": + + def _raise_in_thread(thread: threading.Thread, exc_type: type[BaseException]) -> None: + if not CTYPES_PRESENT: + raise NotImplementedError( + "Sending exceptions to threads is not supported in this Python version." + ) + + if not thread.is_alive(): + raise ValueError("Thread is not alive.") + + thread_id = thread.ident + if thread_id is None: + raise ValueError("Thread has no ident.") + + result = ctypes.pythonapi.PyThreadState_SetAsyncExc( + ctypes.c_ulong(thread_id), + ctypes.py_object(exc_type), + ) + + if result == 0: + raise ValueError("Thread ident is invalid.") + + if result > 1: + ctypes.pythonapi.PyThreadState_SetAsyncExc( + ctypes.c_ulong(thread_id), + None, + ) + raise SystemError("PyThreadState_SetAsyncExc failed.") + + +@overload +def kill_launched_task_prim_sync( + task: LaunchedTask[P, O], *, exception_interval: Union[float, int] = 0.1 +) -> Literal[True]: + """ + Sends exceptions to the underlying concurrency primitive. + May also try to use the recommended way of cancelling the primitive if there is one. + Returns whether the task was actually killed. + """ + + +@overload +def kill_launched_task_prim_sync( + task: LaunchedTask[P, O], + timeout: Union[float, int], + *, + exception_interval: Union[float, int] = 0.1, +) -> bool: + """ + Sends exceptions to the underlying concurrency primitive. + May also try to use the recommended way of cancelling the primitive if there is one. + Returns whether the task was actually killed. + """ + + +def kill_launched_task_prim_sync( + task: LaunchedTask[P, O], + timeout: Optional[Union[float, int]] = None, + *, + exception_interval: Union[float, int] = 0.1, +) -> bool: + has_timeout, timeout_end = ( + (True, time.time() + timeout) if timeout is not None else (False, None) + ) + while (not has_timeout) or (timeout_end is not None and time.time() <= timeout_end): + if not task._thread.is_alive(): + break + _raise_in_thread(task._thread, SystemExit) + time.sleep(exception_interval) + if has_timeout and timeout_end is not None and time.time() > timeout_end: + return False + return True + + +@overload +async def kill_launched_task_prim( + task: LaunchedTask[P, O], *, exception_interval: Union[float, int] = 0.1 +) -> Literal[True]: + """ + Sends exceptions to the underlying concurrency primitive. + May also try to use the recommended way of cancelling the primitive if there is one. + Returns whether the task was actually killed. + """ + + +@overload +async def kill_launched_task_prim( + task: LaunchedTask[P, O], + timeout: Union[float, int], + *, + exception_interval: Union[float, int] = 0.1, +) -> bool: + """ + Sends exceptions to the underlying concurrency primitive. + May also try to use the recommended way of cancelling the primitive if there is one. + Returns whether the task was actually killed. + """ + + +async def kill_launched_task_prim( + task: LaunchedTask[P, O], + timeout: Optional[Union[float, int]] = None, + *, + exception_interval: Union[float, int] = 0.1, +) -> bool: + has_timeout, timeout_end = ( + (True, time.time() + timeout) if timeout is not None else (False, None) + ) + if task._task.cancel(): + return True + while (not has_timeout) or (timeout_end is not None and time.time() <= timeout_end): + if not task._task.done(): + break + task._task.set_exception(SystemExit) + await asyncio.sleep(exception_interval) + if has_timeout and timeout_end is not None and time.time() > timeout_end: + return False + return True + + +# async def task_1(): +# print("Starting task 1...") +# await sleep_prim(2) +# print("Task 1 done.") + + +# async def task_2(msg: Any): +# print("Starting task 2...") +# await sleep_prim(1) +# print("Task 2 says:", msg) +# print("Task 2 done.") + + +# async def task_3(delay: Union[float, int]): +# print("Starting task 3...") +# await sleep_prim(delay) +# print("Task 3 done.") + + +# async def main(): +# await gather_concurrently_prim( +# create_task(task_1), create_task(task_2, msg="Hello there!"), create_task(task_3, 3) +# ) +# print("Launching task...") +# task = await launch_concurrently_prim(create_task(task_3, 5)) +# print("Launched task.") +# await sleep_prim(4) +# print("Joining task...") +# await join_launched_task_prim(task) +# print("Task done.") + + +# if __name__ == "__main__": +# if "IS_ASYNC": +# asyncio.run(main()) +# else: +# main() diff --git a/scratchattach/_core/site/_base.py b/scratchattach/_core/site/_base.py new file mode 100644 index 00000000..bed6eae3 --- /dev/null +++ b/scratchattach/_core/site/_base.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TypeVar, Optional, Self, Union, Any, Generic, TypeAlias, cast, overload, Literal +import json + +if "IS_ASYNC": + from collections.abc import Callable, Awaitable +else: + from collections.abc import Callable + +from scratchattach.utils import exceptions, commons, optional_async +from scratchattach._shared import http as shared_http +from . import session +from ..primitives import http + +D = TypeVar("D") +C = TypeVar("C", bound="BaseSiteComponent") + + +class BaseSiteComponent(ABC, Generic[D]): + _session: session.Session | session.UnauthSession + update_api: str + + # @abstractmethod + # def __init__(self): # dataclasses do not implement __init__ directly + # pass + + async def update(self): + """ + Updates the attributes of the object by performing an API response. Returns True if the update was successful. + """ + async with self._session.http_session.request( + self.update_method, self.update_api, shared_http.options().timeout(10).value + ) as response: + if response.status_code == 429: + return "429" # TODO: Check what this does and replace it + + response_json = await response.json() + + if response_json == {"response": "Too many requests"}: + return "429" + + assert isinstance(response_json, dict) + + if "code" in response_json: + return False + + return self._update_from_data(cast(D, response_json)) + + async def updated(self) -> Self: + await self.update() + return self + + @abstractmethod + def _update_from_data(self, data: D) -> bool: + """ + Parses the API response that is fetched in the update-method. Class specific, must be overridden in classes inheriting from this one. + """ + + def _assert_auth(self) -> session.Session: + if isinstance(self._session, session.UnauthSession): + raise exceptions.Unauthenticated( + "You need to use session.connect_xyz (NOT get_xyz) in order to perform this operation." + ) + return self._session + + @classmethod + def _get_object( + cls, + identificator_name, + identificator, + NotFoundException, + session: session.Session | session.UnauthSession, + ) -> Self: + # Internal function: Generalization of the process ran by get_user, get_studio etc. + # Builds an object of class that is inheriting from BaseSiteComponent + # # Class must inherit from BaseSiteComponent + from scratchattach.site import project + + try: + use_class: type = cls + if cls is project.PartialProject: + use_class = project.Project + assert issubclass(use_class, cls) + _object = use_class(**{identificator_name: identificator, "_session": session}) + r = _object.update() + if r == "429": + raise exceptions.Response429( + "Your network is blocked or rate-limited by Scratch.\n" + "If you're using an online IDE like replit.com, try running the code on your computer." + ) + if not r: + # Target is unshared. The cases that this can happen in are hardcoded: + if cls is project.PartialProject: # Case: Target is an unshared project. + _object = project.PartialProject( + **{identificator_name: identificator, "shared": False, "_session": session} + ) + assert isinstance(_object, cls) + return _object + else: + raise NotFoundException + else: + return _object + except KeyError as e: + raise NotFoundException(f"Key error at key {e} when reading API response") + except Exception as e: + raise e + + def _make_linked_object( + self, identificator_id, identificator, cls: type[C], not_found_exception + ) -> C: + """ + Internal function for making a linked object (authentication kept) based on an identificator (like a project id or username) + Class must inherit from BaseSiteComponent + """ + return cls._get_object(identificator_id, identificator, not_found_exception, self._session) + + @classmethod + def parse_object_list( + cls, + raw: list[D], + /, + session: session.Session | session.UnauthSession, + primary_key: str = "id", + ) -> list[Self]: + results = [] + for raw_dict in raw: + try: + _obj = cls( + **{ + primary_key: cast(dict[str, Any], raw_dict)[primary_key], + "_session": session, + } + ) + # noinspection PyProtectedMember + _obj._update_from_data(raw_dict) + results.append(_obj) + except Exception as e: + print("Warning raised by scratchattach: failed to parse ", raw_dict, "error", e) + return results + + def supply_data_dict(self, data: D) -> bool: + return self._update_from_data(data) + + update_method: shared_http.HTTPMethod = shared_http.HTTPMethod.GET + """ + HTTP method for getting updated information for this component + """ + + +F = TypeVar("F") + +if "IS_ASYNC": # type: ignore[misc] + + @overload + async def api_iterative_data( + fetch_func: Callable[[int, int], Awaitable[list[F] | None]], + limit: int, + offset: int, + max_req_limit: int = 40, + unpack: Literal[True] = True, + ) -> list[F]: ... + + @overload + async def api_iterative_data( + fetch_func: Callable[[int, int], Awaitable[F | None]], + limit: int, + offset: int, + max_req_limit: int = 40, + unpack: Literal[False] = False, + ) -> list[F]: ... +else: + + @overload + def api_iterative_data( + fetch_func: Callable[[int, int], list[F] | None], + limit: int, + offset: int, + max_req_limit: int = 40, + unpack: Literal[True] = True, + ) -> list[F]: ... + + @overload + def api_iterative_data( + fetch_func: Callable[[int, int], F | None], + limit: int, + offset: int, + max_req_limit: int = 40, + unpack: Literal[False] = False, + ) -> list[F]: ... + + +async def api_iterative_data( + fetch_func: Callable[[int, int], Any], + limit: int, + offset: int, + max_req_limit: int = 40, + unpack: bool = True, +) -> list: + """ + Iteratively gets data by calling fetch_func with a moving offset and a limit. + Once fetch_func returns None, the retrieval is completed. + """ + if limit is None: + limit = max_req_limit + + end = offset + limit + api_data: list[Any] = [] + for offs in range(offset, end, max_req_limit): + # Mimic actual scratch by only requesting the max amount + data = await fetch_func(offs, max_req_limit) + if data is None: + break + + if unpack: + api_data.extend(data) + else: + api_data.append(data) + + if len(data) < max_req_limit: + break + + return api_data[:limit] + + +async def api_iterative( + session: session.Session | session.UnauthSession, + url: str, + *, + limit: int, + offset: int, + max_req_limit: int = 40, + add_params: str = "", + _headers: Optional[dict] = None, + cookies: Optional[dict] = None, +) -> list[F]: + """ + Function for getting data from one of Scratch's iterative JSON API endpoints (like /users//followers, or /users//projects) + """ + if _headers is None: + _headers = commons.headers.copy() + if cookies is None: + cookies = {} + + if offset < 0: + raise exceptions.BadRequest("offset parameter must be >= 0") + if limit < 0: + raise exceptions.BadRequest("limit parameter must be >= 0") + + async def fetch(off: int, lim: int) -> list[F] | None: + """ + Performs a single API request + """ + async with session.http_session.get( + f"{url}?limit={lim}&offset={off}{add_params}", + ) as response: + resp = cast( + list[F], + await response.json(), + ) + + if not resp: + return None + if resp == {"code": "BadRequest", "message": ""}: + raise exceptions.BadRequest("The arguments passed are invalid") + return resp + + return await api_iterative_data(fetch, limit, offset, max_req_limit=max_req_limit, unpack=True) diff --git a/scratchattach/_core/site/activity.py b/scratchattach/_core/site/activity.py new file mode 100644 index 00000000..89bfd8ca --- /dev/null +++ b/scratchattach/_core/site/activity.py @@ -0,0 +1,456 @@ +"""Activity and CloudActivity class""" + +from __future__ import annotations + +import html +import warnings + +from dataclasses import dataclass +from typing import Optional, Any +from enum import Enum + +from bs4 import Tag + +from . import user, project, studio, session, forum, comment +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions + + +class ActivityTypes(Enum): + loveproject = "loveproject" + favoriteproject = "favoriteproject" + becomecurator = "becomecurator" + followuser = "followuser" + followstudio = "followstudio" + shareproject = "shareproject" + remixproject = "remixproject" + becomeownerstudio = "becomeownerstudio" + addcomment = "addcomment" + curatorinvite = "curatorinvite" + userjoin = "userjoin" + studioactivity = "studioactivity" + forumpost = "forumpost" + updatestudio = "updatestudio" + createstudio = "createstudio" + promotetomanager = "promotetomanager" + updateprofile = "updateprofile" + removeprojectfromstudio = "removeprojectfromstudio" + addprojecttostudio = "addprojecttostudio" + performaction = "performaction" + + +@dataclass +class Activity(BaseSiteComponent): + """ + Represents a Scratch activity (message or other user page activity) + """ + + _session: Optional[session.Session] = None + raw: Any = None + + id: Optional[int] = None + actor_username: Optional[str] = None + + project_id: Optional[int] = None + gallery_id: Optional[int] = None + username: Optional[str] = None + followed_username: Optional[str] = None + recipient_username: Optional[str] = None + title: Optional[str] = None + project_title: Optional[str] = None + gallery_title: Optional[str] = None + topic_title: Optional[str] = None + topic_id: Optional[int] = None + target_name: Optional[str] = None + target_id: Optional[int | str] = None + + parent_title: Optional[str] = None + parent_id: Optional[int] = None + + comment_type: Optional[int] = None + comment_obj_id: Optional[int] = None + comment_obj_title: Optional[str] = None + comment_id: Optional[int] = None + comment_fragment: Optional[str] = None + + changed_fields: Optional[dict[str, str]] = None + is_reshare: Optional[bool] = None + + datetime_created: Optional[str] = None + time: Any = None + type: Optional[ActivityTypes] = None + + def __repr__(self): + return f"Activity({repr(self.raw)})" + + def __str__(self): + return "-A " + " ".join(self.parts) + + def _parts_simple(self, verb: str, obj: str): + return [str(self.actor_username), verb, obj] + + def _parts_comment(self) -> list[str]: + ret = [str(self.actor_username), "commented on"] + + if self.comment_type not in (0, 1, 2): + raise ValueError(f"Unknown comment type: {self.comment_type}") + ret.append( + { + 0: f"-P {self.comment_obj_title!r} ({self.comment_obj_id}", + 1: f"-U {self.comment_obj_title}", + 2: f"-S {self.comment_obj_title!r} ({self.comment_obj_id}", + }[self.comment_type] + ) + ret[-1] += f"#{self.comment_id})" + + ret.append(str(html.unescape(str(self.comment_fragment)))) + + return ret + + @property + def parts(self): + """ + Return format: [actor username] + N * [action, object] + :return: A list of parts of the message. Join the parts to get a readable version, which is done with str(activity) + """ + SIMPLE_SOLNS = { + ActivityTypes.loveproject: ("loved", f"-P {self.title!r} ({self.project_id})"), + ActivityTypes.favoriteproject: ("favorited", f"-P {self.project_title!r} ({self.project_id})"), + ActivityTypes.becomecurator: ("now curating", f"-S {self.title!r} ({self.gallery_id})"), + ActivityTypes.followuser: ("followed", f"-U {self.followed_username}"), + ActivityTypes.followstudio: ("followed", f"-S {self.title!r} ({self.gallery_id})"), + ActivityTypes.shareproject: ( + "reshared" if self.is_reshare else "shared", + f"-P {self.title!r} ({self.project_id})", + ), + ActivityTypes.remixproject: ( + "remixed", + f"-P {self.parent_title!r} ({self.parent_id}) as -P {self.title!r} ({self.project_id})", + ), + ActivityTypes.becomeownerstudio: ("became owner of", f"-S {self.gallery_title!r} ({self.gallery_id})"), + ActivityTypes.curatorinvite: ("invited you to curate", f"-S {self.title!r} ({self.gallery_id})"), + ActivityTypes.forumpost: ("posted in", f"-F {self.topic_title} ({self.topic_id})"), + ActivityTypes.updatestudio: ("updated", f"-S {self.gallery_title} ({self.gallery_id})"), + ActivityTypes.createstudio: ("created", f"-S {self.gallery_title} ({self.gallery_id})"), + None: (), # to satisfy type checker; () is falsy + } + if args := SIMPLE_SOLNS.get(self.type): + return self._parts_simple(*args) + + match self.type: + case ActivityTypes.addcomment: + return self._parts_comment() + + case ActivityTypes.userjoin: + # This is also the first message you get - 'Welcome to Scratch' + return [str(self.actor_username), "joined Scratch"] + + case ActivityTypes.studioactivity: + # the actor username should be systemuser + return [str(self.actor_username), "Studio activity", "", f"-S {self.title!r} ({self.gallery_id})"] + + case ActivityTypes.promotetomanager: + return [ + str(self.actor_username), + "promoted", + f"-U {self.recipient_username}", + "in", + f"-S {self.gallery_title} ({self.gallery_id})", + ] + + case ActivityTypes.updateprofile: + return [str(self.actor_username), "updated their profile.", f"Changed fields: {self.changed_fields}"] + + case ActivityTypes.removeprojectfromstudio: + return [ + f"{self.actor_username}", + "removed", + f"-P {self.project_title} ({self.project_id})", + "from", + f"-S {self.gallery_title} ({self.gallery_id})", + ] + + case ActivityTypes.addprojecttostudio: + return [ + f"{self.actor_username}", + "added", + f"-P {self.project_title} ({self.project_id})", + "to", + f"-S {self.gallery_title} ({self.gallery_id})", + ] + + case ActivityTypes.performaction: + return [f"{self.actor_username}", "performed an action"] + + case _: + raise NotImplementedError( + f"Activity type {self.type!r} is not implemented!\n" + f"{self.raw=}\n" + f"Raise an issue on github: https://github.com/TimMcCool/scratchattach/issues" + ) + + def update(self): + print("Warning: Activity objects can't be updated") + return False # Objects of this type cannot be updated + + def _update_from_data(self, data): + self.raw = data + + self._session = data.get("_session", self._session) + self.raw = data.get("raw", self.raw) + + self.id = data.get("id", self.id) + self.actor_username = data.get("actor_username", self.actor_username) + + self.project_id = data.get("project_id", self.project_id) + self.gallery_id = data.get("gallery_id", self.gallery_id) + self.username = data.get("username", self.username) + self.followed_username = data.get("followed_username", self.followed_username) + self.recipient_username = data.get("recipient_username", self.recipient_username) + self.title = data.get("title", self.title) + self.project_title = data.get("project_title", self.project_title) + self.gallery_title = data.get("gallery_title", self.gallery_title) + self.topic_title = data.get("topic_title", self.topic_title) + self.topic_id = data.get("topic_id", self.topic_id) + self.target_name = data.get("target_name", self.target_name) + self.target_id = data.get("target_id", self.target_id) + + self.parent_title = data.get("parent_title", self.parent_title) + self.parent_id = data.get("parent_id", self.parent_id) + + self.comment_type = data.get("comment_type", self.comment_type) + self.comment_obj_id = data.get("comment_obj_id", self.comment_obj_id) + self.comment_obj_title = data.get("comment_obj_title", self.comment_obj_title) + self.comment_id = data.get("comment_id", self.comment_id) + self.comment_fragment = data.get("comment_fragment", self.comment_fragment) + + self.changed_fields = data.get("changed_fields", self.changed_fields) + self.is_reshare = data.get("is_reshare", self.is_reshare) + + self.datetime_created = data.get("datetime_created", self.datetime_created) + self.time = data.get("time", self.time) + + _type = data.get("type", self.type) + if _type == "becomehoststudio": + self.type = ActivityTypes.becomeownerstudio + elif _type: + # TODO: do not rely on indexing the enum! I think this is bad practice + self.type = ActivityTypes[_type] + + return True + + def _update_from_json(self, data: dict): + """ + Update using JSON, used in the classroom API. + """ + activity_type = data["type"] + + _time = data.get("datetime_created") + + if "actor" in data: + self.username = data["actor"]["username"] + else: + self.username = data.get("actor_username") + + self.recipient_username = None + if recipient := data.get("recipient"): + self.recipient_username = recipient["username"] + elif ru := data.get("recipient_username"): + self.recipient_username = ru + elif project_creator := data.get("project_creator"): + self.recipient_username = project_creator["username"] + + # Even if `activity_type` is an invalid value; it will default to 'user performed an action' + self.actor_username = self.username + self.raw = data + self.datetime_created = _time + + # NOTE: some type values are treated the same here + # this is by design. the scratch HTML does this using a switch statement + self.type = { + 0: ActivityTypes.followuser, + 1: ActivityTypes.followstudio, + 2: ActivityTypes.loveproject, + 3: ActivityTypes.favoriteproject, + 7: ActivityTypes.addprojecttostudio, + 8: ActivityTypes.shareproject, + 9: ActivityTypes.shareproject, + 10: ActivityTypes.shareproject, + 11: ActivityTypes.remixproject, + # type 12 does not exist in the HTML. That's why it was removed, not merged with type 13. + 13: ActivityTypes.createstudio, + 15: ActivityTypes.updatestudio, + 16: ActivityTypes.removeprojectfromstudio, + 17: ActivityTypes.removeprojectfromstudio, + 18: ActivityTypes.removeprojectfromstudio, + 19: ActivityTypes.removeprojectfromstudio, + 20: ActivityTypes.promotetomanager, + 21: ActivityTypes.promotetomanager, + 22: ActivityTypes.promotetomanager, + 23: ActivityTypes.updateprofile, + 24: ActivityTypes.updateprofile, + 25: ActivityTypes.updateprofile, + 26: ActivityTypes.addcomment, + 27: ActivityTypes.addcomment, + None: ActivityTypes.performaction, # this one is just to satisfy type checkers + }.get(activity_type, ActivityTypes.performaction) + + self.followed_username = data.get("followed_username", self.followed_username) + self.gallery_id = data.get("gallery", self.gallery_id) + self.project_id = data.get("project", self.project_id) + self.is_reshare = data.get("is_reshare", self.is_reshare) + self.comment_fragment = data.get("comment_fragment", self.comment_fragment) + self.comment_type = data.get("comment_type", self.comment_type) + self.comment_obj_id = data.get("comment_obj_id", self.comment_obj_id) + self.comment_obj_title = data.get("comment_obj_title", self.comment_obj_title) + self.comment_id = data.get("comment_id", self.comment_id) + self.parent_id = data.get("parent", self.parent_id) + if self.parent_id: + # activity_type 11 + warnings.warn( + f"This may be incorrectly implemented.\n" + f"Raw data: {data}\n" + f"Please raise an issue on gh: https://github.com/TimMcCool/scratchattach/issues" + ) + if self.type == ActivityTypes.updateprofile: + self.changed_fields = data.get("changed_fields", {}) + + def _update_from_html(self, data: Tag): + + self.raw = data + + _time = data.find("div").find("span").find_next().find_next().text.strip() + + if "\xa0" in _time: + while "\xa0" in _time: + _time = _time.replace("\xa0", " ") + + self.datetime_created = _time + self.actor_username = data.find("div").find("span").text + + self.target_name = data.find("div").find("span").find_next().text + self.target_link = data.find("div").find("span").find_next()["href"] + # note that target_id can also be a username, so it isn't exclusively an int + self.target_id = data.find("div").find("span").find_next()["href"].split("/")[-2] + + _type = data.find("div").find_all("span")[0].next_sibling.strip() + if _type == "loved": + self.type = ActivityTypes.loveproject + + elif _type == "favorited": + self.type = ActivityTypes.favoriteproject + + elif "curator" in _type: + self.type = ActivityTypes.becomecurator + + elif "shared" in _type: + self.type = ActivityTypes.shareproject + + elif "is now following" in _type: + if "users" in self.target_link: + self.type = ActivityTypes.followuser + else: + self.type = ActivityTypes.followstudio + + return True + + def actor(self): + """ + Returns the user that performed the activity as User object + """ + return self._make_linked_object("username", self.actor_username, user.User, exceptions.UserNotFound) + + def target_project(self) -> Optional[project.Project]: + if self.target_id: + return self._make_linked_object("id", self.target_id, project.Project, exceptions.ProjectNotFound) + if self.project_id: + return self._make_linked_object("id", self.project_id, project.Project, exceptions.ProjectNotFound) + return None + + def target_studio(self) -> Optional[studio.Studio]: + if self.target_id: + return self._make_linked_object("id", self.target_id, studio.Studio, exceptions.StudioNotFound) + if self.gallery_id: + return self._make_linked_object("id", self.gallery_id, studio.Studio, exceptions.StudioNotFound) + return None + + def target_user(self) -> Optional[user.User]: + if self.username: + return self._make_linked_object("username", self.username, user.User, exceptions.UserNotFound) + if self.target_name: + return self._make_linked_object("username", self.target_name, user.User, exceptions.UserNotFound) + if self.followed_username: + return self._make_linked_object("username", self.followed_username, user.User, exceptions.UserNotFound) + if self.recipient_username: + return self._make_linked_object("username", self.recipient_username, user.User, exceptions.UserNotFound) + return None + + def target_comment(self) -> Optional[comment.Comment]: + # TODO: make use of self.target_project/target_user/target_studio here. + # Also why is there no use of studio here??? This needs to be tested + if self.comment_type == 0: + if self.comment_obj_id is None: + return None + + # we need author name, but it has not been saved in this object + if self._session is not None: + _proj = self._session.connect_project(self.comment_obj_id) + else: + _proj = project.Project(id=self.comment_obj_id) + + return _proj.comment_by_id(self.comment_id) + + elif self.comment_type == 1: + return user.User(username=self.comment_obj_title, _session=self._session).comment_by_id(self.comment_id) + elif self.comment_type == 2: + return user.User(id=self.comment_obj_id, _session=self._session).comment_by_id(self.comment_id) + else: + return None + + def target(self): + """ + Returns the activity's target (depending on the activity, this is either a User, Project, Studio or Comment object). + May also return None if the activity type is unknown. + """ + if self.type is None: + return None + + _type = self.type.value + if self.type in ( + ActivityTypes.addprojecttostudio, + ActivityTypes.favoriteproject, + ActivityTypes.loveproject, + ActivityTypes.remixproject, + ActivityTypes.removeprojectfromstudio, + ActivityTypes.shareproject, + ): # target is a project + return self.target_project() + + if self.type in (ActivityTypes.becomecurator, ActivityTypes.followstudio): # target is a studio + if ret := self.target_studio(): + return ret + # NOTE: the "becomecurator" type is ambigous - if it is inside the studio activity tab, the target is the user who joined + return self.target_user() + + if ( + self.type + in ( + ActivityTypes.followuser, + ActivityTypes.curatorinvite, + ) + or self.recipient_username + ): # target is a user + # NOTE: the recipient_username field always indicates the target is a user + return self.target_user() + + if self.type == ActivityTypes.addcomment: # target is a comment + if ret := self.target_comment(): + return ret + + raise ValueError(f"Either {self.comment_type} is an invalid comment type, or the linked target could not be found") + + if _type == "forumpost": + # FIXME: why is the id here constant?!?? + return forum.ForumTopic(id=603418, _session=self._session, title=self.title) + + return None diff --git a/scratchattach/_core/site/alert.py b/scratchattach/_core/site/alert.py new file mode 100644 index 00000000..c419f9ce --- /dev/null +++ b/scratchattach/_core/site/alert.py @@ -0,0 +1,226 @@ +# classroom alerts (& normal alerts in the future) + +from __future__ import annotations + +import json +import pprint +import warnings +from dataclasses import dataclass, field, KW_ONLY +from datetime import datetime +from typing_extensions import TYPE_CHECKING, Any, Optional, Union, Self + +from . import user, project, studio, comment, session +from scratchattach.utils import enums + +if TYPE_CHECKING: + ... + + +# todo: implement regular alerts +# If you implement regular alerts, it may be applicable to make EducatorAlert a subclass. + + +@dataclass +class EducatorAlert: + """ + Represents an alert for student activity, viewable at https://scratch.mit.edu/site-api/classrooms/alerts/ + + Attributes: + model: The type of alert (presumably); should always equal "educators.educatoralert" in this class + type: An integer that identifies the type of alert, differentiating e.g. against bans or autoban or censored comments etc + raw: The raw JSON data from the API + id: The ID of the alert (internally called 'pk' by scratch, not sure what this is for) + time_read: The time the alert was read + time_created: The time the alert was created + target: The user that the alert is about (the student) + actor: The user that created the alert (the admin) + target_object: The object that the alert is about (e.g. a project, studio, or comment) + notification_type: not sure what this is for, but inferred from the scratch HTML reference + """ + _: KW_ONLY + # required attrs + target: user.User + actor: user.User + target_object: Optional[Union[project.Project, studio.Studio, comment.Comment, studio.Studio]] + notification_type: str + _session: Optional[session.Session] + + # defaulted attrs + model: str = "educators.educatoralert" + type: int = -1 + raw: dict = field(repr=False, default_factory=dict) + id: int = -1 + time_read: datetime = datetime.fromtimestamp(0.0) + time_created: datetime = datetime.fromtimestamp(0.0) + + + @classmethod + def from_json(cls, data: dict[str, Any], _session: Optional[session.Session] = None) -> Self: + """ + Load an EducatorAlert from a JSON object. + + Arguments: + data (dict): The JSON object + _session (session.Session): The session object used to load this data, to 'connect' to the alerts rather than just 'get' them + + Returns: + EducatorAlert: The loaded EducatorAlert object + """ + model = data.get("model") # With this class, should be equal to educators.educatoralert + assert isinstance(model, str) + alert_id = data.get("pk") # not sure what kind of pk/id this is. Doesn't seem to be a user or class id. + assert isinstance(alert_id, int) + + fields = data.get("fields") + assert isinstance(fields, dict) + + time_read_raw = fields.get("educator_datetime_read") + assert isinstance(time_read_raw, str) + time_read: datetime = datetime.fromisoformat(time_read_raw) + + admin_action = fields.get("admin_action") + assert isinstance(admin_action, dict) + + time_created_raw = admin_action.get("datetime_created") + assert isinstance(time_created_raw, str) + time_created: datetime = datetime.fromisoformat(time_created_raw) + + alert_type = admin_action.get("type") + assert isinstance(alert_type, int) + + target_data = admin_action.get("target_user") + assert isinstance(target_data, dict) + target = user.User(username=target_data.get("username"), + id=target_data.get("pk"), + icon_url=target_data.get("thumbnail_url"), + admin=target_data.get("admin", False), + _session=_session) + + actor_data = admin_action.get("actor") + assert isinstance(actor_data, dict) + actor = user.User(username=actor_data.get("username"), + id=actor_data.get("pk"), + icon_url=actor_data.get("thumbnail_url"), + admin=actor_data.get("admin", False), + _session=_session) + + object_id = admin_action.get("object_id") # this could be a comment id, a project id, etc. + assert isinstance(object_id, int) + target_object: project.Project | studio.Studio | comment.Comment | None = None + + extra_data: dict[str, Any] = json.loads(admin_action.get("extra_data", "{}")) + # todo: if possible, properly implement the incomplete parts of this parser (look for warning.warn()) + notification_type: str = "" + + if "project_title" in extra_data: + # project + target_object = project.Project(id=object_id, + title=extra_data["project_title"], + _session=_session) + elif "comment_content" in extra_data: + # comment + comment_data: dict[str, Any] = extra_data["comment_content"] + content: str | None = comment_data.get("content") + + comment_obj_id: int | None = comment_data.get("comment_obj_id") + + comment_type: int | None = comment_data.get("comment_type") + + if comment_type == 0: + # project + comment_source_type = comment.CommentSource.PROJECT + elif comment_type == 1: + # profile + comment_source_type = comment.CommentSource.USER_PROFILE + else: + # probably a studio + comment_source_type = comment.CommentSource.STUDIO + warnings.warn( + f"The parser was not able to recognise the \"comment_type\" of {comment_type} in the alert JSON response.\n" + f"Full response: \n{pprint.pformat(data)}.\n\n" + f"Please draft an issue on github: https://github.com/TimMcCool/scratchattach/issues, providing this " + f"whole error message. This will allow us to implement an incomplete part of this parser") + + # the comment_obj's corresponding attribute of comment.Comment is the place() method. As it has no cache, the title data is wasted. + # if the comment_obj is deleted, this is still a valid way of working out the title/username + + target_object = comment.Comment( + id=object_id, + content=content, + source=comment_source_type, + source_id=comment_obj_id, + _session=_session + ) + + elif "gallery_title" in extra_data: + # studio + # possible implemented incorrectly + target_object = studio.Studio( + id=object_id, + title=extra_data["gallery_title"], + _session=_session + ) + elif "notification_type" in extra_data: + # possible implemented incorrectly + notification_type = extra_data["notification_type"] + else: + warnings.warn( + f"The parser was not able to recognise the \"extra_data\" in the alert JSON response.\n" + f"Full response: \n{pprint.pformat(data)}.\n\n" + f"Please draft an issue on github: https://github.com/TimMcCool/scratchattach/issues, providing this " + f"whole error message. This will allow us to implement an incomplete part of this parser") + + return cls( + id=alert_id, + model=model, + type=alert_type, + raw=data, + time_read=time_read, + time_created=time_created, + target=target, + actor=actor, + target_object=target_object, + notification_type=notification_type, + _session=_session + ) + + def __str__(self): + return f"EducatorAlert: {self.message}" + + @property + def alert_type(self) -> enums.AlertType: + """ + Get an associated AlertType object for this alert (based on the type index) + """ + alert_type = enums.AlertTypes.find(self.type) + if not alert_type: + alert_type = enums.AlertTypes.default.value + + return alert_type + + @property + def message(self): + """ + Format the alert message using the alert type's message template, as it would be on the website. + """ + raw_message = self.alert_type.message + comment_content = "" + if isinstance(self.target_object, comment.Comment): + comment_content = self.target_object.content + + return raw_message.format(username=self.target.username, + project=self.target_object_title, + studio=self.target_object_title, + notification_type=self.notification_type, + comment=comment_content) + + @property + def target_object_title(self): + """ + Get the title of the target object (if applicable) + """ + if isinstance(self.target_object, project.Project): + return self.target_object.title + if isinstance(self.target_object, studio.Studio): + return self.target_object.title + return None # explicit diff --git a/scratchattach/_core/site/backpack_asset.py b/scratchattach/_core/site/backpack_asset.py new file mode 100644 index 00000000..ecfc2323 --- /dev/null +++ b/scratchattach/_core/site/backpack_asset.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import json +import time +import logging +import warnings + +from dataclasses import dataclass +from typing import Any, TYPE_CHECKING + +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions +from scratchattach.utils.requests import requests + +if TYPE_CHECKING: + from scratchattach import session + + +@dataclass +class BackpackAsset(BaseSiteComponent): + """ + Represents an asset from the backpack. + + Attributes: + + :.id: + + :.type: The asset type (costume, script etc.) + + :.mime: The format in which the content of the backpack asset is saved + + :.name: The name of the backpack asset + + :.filename: Filename of the file containing the content of the backpack asset + + :.thumbnail_url: Link that leads to the asset's thumbnail (the image shown in the backpack UI) + + :.download_url: Link that leads to a file containing the content of the backpack asset + """ + + id: str + _session: session.Session | None = None + type: str | None = None + mime: str | None = None + name: str | None = None + filename: str | None = None + thumbnail_url: str | None = None + download_url: str | None = None + + def __repr__(self) -> str: + return f"BackpackAsset({self.filename})" + + def update(self): + warnings.warn("Warning: BackpackAsset objects can't be updated") + return False # Objects of this type cannot be updated + + def _update_from_data(self, data: dict[str, str]) -> bool: + self.id = data.get("id", self.id) + self.type = data.get("type", self.type) + self.mime = data.get("mime", self.mime) + self.name = data.get("name", self.name) + self.filename = data.get("body", self.filename) + if "thumbnail" in data: + self.thumbnail_url = "https://backpack.scratch.mit.edu/" + data["thumbnail"] + if "body" in data: + self.download_url = "https://backpack.scratch.mit.edu/" + data["body"] + return True + + @property + def _data_bytes(self) -> bytes: + try: + with requests.no_error_handling(): + return requests.get(self.download_url).content + except Exception as e: + raise exceptions.FetchError(f"Failed to download asset: {e}") + + @property + def file_ext(self): + return self.filename.split(".")[-1] + + @property + def is_json(self): + return self.file_ext == "json" + + @property + def data(self) -> dict | list | int | None | str | bytes | float: + if self.is_json: + return json.loads(self._data_bytes) + else: + # It's either a zip + return self._data_bytes + + def download(self, *, fp: str = ""): + """ + Downloads the asset content to the given directory. The given filename is equal to the value saved in the .filename attribute. + + Args: + fp (str): The path of the directory the file will be saved in. + """ + if not (fp.endswith("/") or fp.endswith("\\")): + fp = fp + "/" + open(f"{fp}{self.filename}", "wb").write(self._data_bytes) + + def delete(self): + self._assert_auth() + + return requests.delete( + f"https://backpack.scratch.mit.edu/{self._session.username}/{self.id}", + headers=self._session._headers, + timeout=10, + ).json() diff --git a/scratchattach/_core/site/browser_cookie3_stub.py b/scratchattach/_core/site/browser_cookie3_stub.py new file mode 100644 index 00000000..9d63d7df --- /dev/null +++ b/scratchattach/_core/site/browser_cookie3_stub.py @@ -0,0 +1,17 @@ +# browser_cookie3.pyi + +import http.cookiejar +from typing import Optional + +def chrome(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: return NotImplemented +def chromium(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: return NotImplemented +def firefox(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: return NotImplemented +def opera(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: return NotImplemented +def edge(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: return NotImplemented +def brave(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: return NotImplemented +def vivaldi(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: return NotImplemented +def safari(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: return NotImplemented +def lynx(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: return NotImplemented +def w3m(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: return NotImplemented + +def load() -> http.cookiejar.CookieJar: return NotImplemented diff --git a/scratchattach/_core/site/browser_cookies.py b/scratchattach/_core/site/browser_cookies.py new file mode 100644 index 00000000..ba784a66 --- /dev/null +++ b/scratchattach/_core/site/browser_cookies.py @@ -0,0 +1,61 @@ +from typing import Optional, TYPE_CHECKING +from typing_extensions import assert_never +from http.cookiejar import CookieJar +from enum import Enum, auto +browsercookie_err = None +try: + if TYPE_CHECKING: + from . import browser_cookie3_stub as browser_cookie3 + else: + import browser_cookie3 +except Exception as e: + browsercookie = None + browsercookie_err = e + +class Browser(Enum): + ANY = auto() + FIREFOX = auto() + CHROME = auto() + EDGE = auto() + SAFARI = auto() + CHROMIUM = auto() + VIVALDI = auto() + EDGE_DEV = auto() + + +FIREFOX = Browser.FIREFOX +CHROME = Browser.CHROME +EDGE = Browser.EDGE +SAFARI = Browser.SAFARI +CHROMIUM = Browser.CHROMIUM +VIVALDI = Browser.VIVALDI +ANY = Browser.ANY +EDGE_DEV = Browser.EDGE_DEV + +def cookies_from_browser(browser : Browser = ANY) -> dict[str, str]: + """ + Import cookies from browser to login + """ + if not browser_cookie3: + raise browsercookie_err or ModuleNotFoundError() + cookies : Optional[CookieJar] = None + if browser is Browser.ANY: + cookies = browser_cookie3.load() + elif browser is Browser.FIREFOX: + cookies = browser_cookie3.firefox() + elif browser is Browser.CHROME: + cookies = browser_cookie3.chrome() + elif browser is Browser.EDGE: + cookies = browser_cookie3.edge() + elif browser is Browser.SAFARI: + cookies = browser_cookie3.safari() + elif browser is Browser.CHROMIUM: + cookies = browser_cookie3.chromium() + elif browser is Browser.VIVALDI: + cookies = browser_cookie3.vivaldi() + elif browser is Browser.EDGE_DEV: + raise ValueError("EDGE_DEV is not supported anymore.") + else: + assert_never(browser) + assert isinstance(cookies, CookieJar) + return {cookie.name: cookie.value for cookie in cookies if "scratch.mit.edu" in cookie.domain and cookie.value} \ No newline at end of file diff --git a/scratchattach/_core/site/classroom.py b/scratchattach/_core/site/classroom.py new file mode 100644 index 00000000..c2258802 --- /dev/null +++ b/scratchattach/_core/site/classroom.py @@ -0,0 +1,454 @@ +from __future__ import annotations + +import json +import warnings +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional, TYPE_CHECKING, Any, Callable + +import bs4 +from bs4 import BeautifulSoup + +if TYPE_CHECKING: + from scratchattach.site.session import Session + +from scratchattach.utils.commons import requests +from . import user, activity, typed_dicts +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions, commons +from scratchattach.utils.commons import headers + + +@dataclass +class Classroom(BaseSiteComponent): + title: str = "" + id: int = 0 + classtoken: str = "" + + author: Optional[user.User] = None + about_class: str = "" + working_on: str = "" + + is_closed: bool = False + datetime: datetime = datetime.fromtimestamp(0.0) + + + update_function: Callable = field(repr=False, default=requests.get) + _session: Optional[Session] = field(repr=False, default=None) + + def __post_init__(self): + # Info on how the .update method has to fetch the data: + # NOTE: THIS DOESN'T WORK WITH CLOSED CLASSES! + if self.id: + self.update_api = f"https://api.scratch.mit.edu/classrooms/{self.id}" + elif self.classtoken: + self.update_api = f"https://api.scratch.mit.edu/classtoken/{self.classtoken}" + else: + raise KeyError(f"No class id or token provided! {self.__dict__ = }") + + # Headers and cookies: + if self._session is None: + self._headers = commons.headers + self._cookies = {} + else: + self._headers = self._session._headers + self._cookies = self._session._cookies + + # Headers for operations that require accept and Content-Type fields: + self._json_headers = {**self._headers, + "accept": "application/json", + "Content-Type": "application/json"} + + def __str__(self) -> str: + return f"" + + def update(self): + try: + success = super().update() + except exceptions.ClassroomNotFound: + success = False + + if not success: + response = requests.get(f"https://scratch.mit.edu/classes/{self.id}/") + soup = BeautifulSoup(response.text, "html.parser") + + headings = soup.find_all("h1") + for heading in headings: + if heading.text == "Whoops! Our server is Scratch'ing its head": + raise exceptions.ClassroomNotFound(f"Classroom id {self.id} is not closed and cannot be found.") + + # id, title, description, status, date_start (iso format), educator/username + + title = soup.find("title").contents[0][:-len(" on Scratch")] + + overviews = soup.find_all("p", {"class": "overview"}) + description, status = overviews[0].text, overviews[1].text + + educator_username = None + pfx = "Scratch.INIT_DATA.PROFILE = {\n model: {\n id: '" + sfx = "',\n userId: " + for script in soup.find_all("script"): + if pfx in script.text: + educator_username = commons.webscrape_count(script.text, pfx, sfx, str) + + ret: typed_dicts.ClassroomDict = { + "id": self.id, + "title": title, + "description": description, + "educator": {}, + "status": status, + "is_closed": True + } + + if educator_username: + ret["educator"]["username"] = educator_username + + return self._update_from_data(ret) + return success + + def _update_from_data(self, data: typed_dicts.ClassroomDict): + self.id = int(data["id"]) + self.title = data["title"] + self.about_class = data["description"] + self.working_on = data["status"] + self.datetime = datetime.fromisoformat(data["date_start"]) + self.author = user.User(username=data["educator"]["username"], _session=self._session) + self.author.supply_data_dict(data["educator"]) + self.is_closed = bool(data["date_end"]) + return True + + def student_count(self) -> int: + # student count + text = requests.get( + f"https://scratch.mit.edu/classes/{self.id}/", + headers=self._headers + ).text + return commons.webscrape_count(text, "Students (", ")") + + def student_names(self, *, page=1) -> list[str]: + """ + Returns the student on the class. + + Keyword Arguments: + page: The page of the students that should be returned. + + Returns: + list: The usernames of the class students + """ + if self.is_closed: + ret = [] + response = requests.get(f"https://scratch.mit.edu/classes/{self.id}/") + soup = BeautifulSoup(response.text, "html.parser") + found = set("") + + for result in soup.css.select("ul.scroll-content .user a"): + result_text = result.text.strip() + if result_text in found: + continue + found.add(result_text) + ret.append(result_text) + + # for scrollable in soup.find_all("ul", {"class": "scroll-content"}): + # if not isinstance(scrollable, Tag): + # continue + # for item in scrollable.contents: + # if not isinstance(item, bs4.NavigableString): + # if "user" in item.attrs["class"]: + # anchors = item.find_all("a") + # if len(anchors) == 2: + # ret.append(anchors[1].text.strip()) + + return ret + + text = requests.get( + f"https://scratch.mit.edu/classes/{self.id}/students/?page={page}", + headers=self._headers + ).text + textlist = [i.split('/">')[0] for i in text.split(' list[int]: + """ + Returns the class studio on the class. + + Keyword Arguments: + page: The page of the students that should be returned. + + Returns: + list: The id of the class studios + """ + if self.is_closed: + ret = [] + response = requests.get(f"https://scratch.mit.edu/classes/{self.id}/") + soup = BeautifulSoup(response.text, "html.parser") + + for result in soup.css.select("ul.scroll-content .gallery a[href]:not([class])"): + value = result["href"] + if not isinstance(value, str): + value = value[0] + ret.append(commons.webscrape_count(value, "/studios/", "/")) + + # for scrollable in soup.find_all("ul", {"class": "scroll-content"}): + # for item in scrollable.contents: + # if not isinstance(item, bs4.NavigableString): + # if "gallery" in item.attrs["class"]: + # anchor = item.find("a") + # if "href" in anchor.attrs: + # ret.append(commons.webscrape_count(anchor.attrs["href"], "/studios/", "/")) + return ret + + text = requests.get( + f"https://scratch.mit.edu/classes/{self.id}/studios/?page={page}", + headers=self._headers + ).text + textlist = [int(i.split('/">')[0]) for i in text.split('\n None: + self._check_session() + requests.post(f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, cookies=self._cookies, + files={"file": thumbnail}) + + def set_description(self, desc: str) -> None: + self._check_session() + response = requests.put(f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, cookies=self._cookies, + json={"description": desc}) + + try: + data = response.json() + if data["description"] == desc: + # Success! + return + else: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def set_working_on(self, status: str) -> None: + self._check_session() + response = requests.put(f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, cookies=self._cookies, + json={"status": status}) + + try: + data = response.json() + if data["status"] == status: + # Success! + return + else: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def set_title(self, title: str) -> None: + self._check_session() + response = requests.put(f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, cookies=self._cookies, + json={"title": title}) + + try: + data = response.json() + if data["title"] == title: + # Success! + return + else: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def add_studio(self, name: str, description: str = '') -> None: + self._check_session() + requests.post("https://scratch.mit.edu/classes/create_classroom_gallery/", + json={ + "classroom_id": str(self.id), + "classroom_token": self.classtoken, + "title": name, + "description": description}, + headers=self._headers, cookies=self._cookies) + + def reopen(self) -> None: + self._check_session() + response = requests.put(f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, cookies=self._cookies, + json={"visibility": "visible"}) + + try: + response.json() + + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def close(self) -> None: + self._check_session() + response = requests.post(f"https://scratch.mit.edu/site-api/classrooms/close_classroom/{self.id}/", + headers=self._headers, cookies=self._cookies) + + try: + response.json() + + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def register_student(self, username: str, password: str = '', birth_month: Optional[int] = None, + birth_year: Optional[int] = None, + gender: Optional[str] = None, country: Optional[str] = None, is_robot: bool = False) -> None: + return register_by_token(self.id, self.classtoken, username, password, birth_month or 1, birth_year or 2000, gender or "(Prefer not to say)", country or "United+States", + is_robot) + + def generate_signup_link(self): + if self.classtoken is not None: + return f"https://scratch.mit.edu/signup/{self.classtoken}" + + self._check_session() + + response = requests.get(f"https://scratch.mit.edu/site-api/classrooms/generate_registration_link/{self.id}/", + headers=self._headers, cookies=self._cookies) + # Should really check for '404' page + data = response.json() + if "reg_link" in data: + return data["reg_link"] + else: + raise exceptions.Unauthorized(f"{self._session} is not authorised to generate a signup link of {self}") + + def public_activity(self, *, limit=20): + """ + Returns: + list: The user's activity data as parsed list of scratchattach.activity.Activity objects + """ + if limit > 20: + warnings.warn("The limit is set to more than 20. There may be an error") + soup = BeautifulSoup( + requests.get(f"https://scratch.mit.edu/site-api/classrooms/activity/public/{self.id}/?limit={limit}").text, + 'html.parser') + + activities = [] + source = soup.find_all("li") + + for data in source: + _activity = activity.Activity(_session=self._session, raw=data) + _activity._update_from_html(data) + activities.append(_activity) + + return activities + + def activity(self, student: str = "all", mode: str = "Last created", page: Optional[int] = None) -> list[activity.Activity]: + """ + Get a list of private activity, only available to the class owner. + Returns: + list The private activity of users in the class + """ + + self._check_session() + + ascsort, descsort = commons.get_class_sort_mode(mode) + + with requests.no_error_handling(): + try: + data = requests.get(f"https://scratch.mit.edu/site-api/classrooms/activity/{self.id}/{student}/", + params={"page": page, "ascsort": ascsort, "descsort": descsort}, + headers=self._headers, cookies=self._cookies).json() + except json.JSONDecodeError: + return [] + + _activity: list[activity.Activity] = [] + for activity_json in data: + _activity.append(activity.Activity(_session=self._session)) + _activity[-1]._update_from_json(activity_json) # NOT the same as _update_from_data + + return _activity + + +def get_classroom(class_id: str) -> Classroom: + """ + Gets a class without logging in. + + Args: + class_id (str): class id of the requested class + + Returns: + scratchattach.classroom.Classroom: An object representing the requested classroom + + Warning: + Any methods that require authentication will not work on the returned object. + + If you want to use these, get the user with :meth:`scratchattach.session.Session.connect_classroom` instead. + """ + warnings.warn( + "For methods that require authentication, use session.connect_classroom instead of get_classroom\n" + "If you want to remove this warning, use warnings.filterwarnings('ignore', category=scratchattach.ClassroomAuthenticationWarning)\n" + "To ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use " + "`warnings.filterwarnings('ignore', category=scratchattach.GetAuthenticationWarning)`.", + exceptions.ClassroomAuthenticationWarning + ) + return commons._get_object("id", class_id, Classroom, exceptions.ClassroomNotFound) + + +def get_classroom_from_token(class_token) -> Classroom: + """ + Gets a class without logging in. + + Args: + class_token (str): class token of the requested class + + Returns: + scratchattach.classroom.Classroom: An object representing the requested classroom + + Warning: + Any methods that require authentication will not work on the returned object. + + If you want to use these, get the user with :meth:`scratchattach.session.Session.connect_classroom` instead. + """ + warnings.warn( + "For methods that require authentication, use session.connect_classroom instead of get_classroom. " + "If you want to remove this warning, use warnings.filterwarnings('ignore', category=ClassroomAuthenticationWarning). " + "To ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use " + "warnings.filterwarnings('ignore', category=GetAuthenticationWarning).", + exceptions.ClassroomAuthenticationWarning + ) + return commons._get_object("classtoken", class_token, Classroom, exceptions.ClassroomNotFound) + + +def register_by_token(class_id: int, class_token: str, username: str, password: str, birth_month: int, birth_year: int, + gender: str, country: str, is_robot: bool = False) -> None: + data = {"classroom_id": class_id, + "classroom_token": class_token, + + "username": username, + "password": password, + "birth_month": birth_month, + "birth_year": birth_year, + "gender": gender, + "country": country, + "is_robot": is_robot} + + response = requests.post("https://scratch.mit.edu/classes/register_new_student/", + data=data, headers=commons.headers, cookies={"scratchcsrftoken": 'a'}) + ret = response.json()[0] + + if "username" in ret: + return + else: + raise exceptions.Unauthorized(f"Can't create account: {response.text}") diff --git a/scratchattach/_core/site/cloud_activity.py b/scratchattach/_core/site/cloud_activity.py new file mode 100644 index 00000000..4be43fe5 --- /dev/null +++ b/scratchattach/_core/site/cloud_activity.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import time +from typing import Union, TypeGuard, Optional +from dataclasses import dataclass, field +import warnings + +from scratchattach.cloud import _base +from scratchattach.utils import exceptions +from scratchattach.site import user +from scratchattach.site import project as project_module +from ._base import BaseSiteComponent +from . import typed_dicts, session + +@dataclass +class CloudActivity(BaseSiteComponent[Union[typed_dicts.CloudActivityDict, typed_dicts.CloudLogActivityDict]]): + """ + Represents a cloud activity (a cloud variable set / creation / deletion). + """ + username: str = field(kw_only=True, default="") + "The user who caused the cloud event (the user who added / set / deleted the cloud variable)" + var: str = field(kw_only=True, default="") + "The name of the cloud variable that was updated (specified without the cloud emoji)" + name: str = field(kw_only=True, default="") + "The name of the cloud variable that was updated (specified without the cloud emoji)" + actual_var: str = field(kw_only=True, default="") + type: str = field(kw_only=True, default="set") + "The activity type" + timestamp: float = field(kw_only=True, default=0.0) + "Then timestamp of when the action was performed" + value: Union[float, int, str] = field(kw_only=True, default="0.0") + "If the cloud variable was set, then this attribute provides the value the cloud variable was set to" + cloud: _base.AnyCloud = field(kw_only=True, default_factory=lambda : _base.DummyCloud()) + "The cloud (as object inheriting from scratchattach.Cloud.BaseCloud) that the cloud activity corresponds to" + _session: Optional[session.Session] = field(kw_only=True, default=None) + + # def __init__(self, **entries): + # # Set attributes every CloudActivity object needs to have: + # self._session = None + # self.cloud = None + # self.user = None + # self.username = None + # self.type = None + # self.timestamp = time.time() + + # # Update attributes from entries dict: + # self.__dict__.update(entries) + + def update(self): + warnings.warn("CloudActivity objects can't be updated", exceptions.InvalidUpdateWarning) + return False # Objects of this type cannot be updated + + def __eq__(self, activity2): + # CloudLogEvents needs to check if two activites are equal (to finde new ones), therefore CloudActivity objects need to be comparable + return self.user == activity2.user and self.type == activity2.type and self.timestamp == activity2.timestamp and self.value == activity2.value and self.name == activity2.name + + def _update_from_data(self, data: Union[typed_dicts.CloudActivityDict, typed_dicts.CloudLogActivityDict]) -> bool: + def is_cloud_log_activity(activity: Union[typed_dicts.CloudActivityDict, typed_dicts.CloudLogActivityDict]) -> TypeGuard[typed_dicts.CloudLogActivityDict]: + return "verb" in activity + def is_cloud_activity(activity: Union[typed_dicts.CloudActivityDict, typed_dicts.CloudLogActivityDict]) -> TypeGuard[typed_dicts.CloudActivityDict]: + return "method" in activity + self.name = data["name"] + self.var = data["name"] + self.value = data["value"] + self.actual_var = data.get("variable_name") or self.var + if is_cloud_log_activity(data): + self.user = data["user"] + self.username = data["user"] + self.timestamp = data["timestamp"] + self.type = data["verb"].removesuffix("_var") + elif is_cloud_activity(data): + self.type = data["method"] + if "cloud" in data: + self.cloud = data["cloud"] + return True + + def a(self, **k): + pass + + def load_log_data(self): + if self.cloud is None: + print("Warning: There aren't cloud logs available for this cloud, therefore the user and exact timestamp can't be loaded") + else: + if isinstance(self.cloud, _base.LogCloud): + logs = self.cloud.logs(filter_by_var_named=self.var, limit=100) + matching = list(filter(lambda x: x.value == self.value and x.timestamp <= self.timestamp, logs)) + if matching == []: + return False + activity = matching[0] + self.username = activity.username + self.user = activity.username + self.timestamp = activity.timestamp + return True + else: + print("Warning: There aren't cloud logs available for this cloud, therefore the user and exact timestamp can't be loaded") + return False + + def actor(self): + """ + Returns the user that performed the cloud activity as scratchattach.user.User object + """ + if self.username is None: + return None + return self._make_linked_object("username", self.username, user.User, exceptions.UserNotFound) + + def project(self) -> Optional[project_module.Project]: + """ + Returns the project where the cloud activity was performed as scratchattach.project.Project object + """ + def make_linked(cloud: _base.BaseCloud) -> project_module.Project: + return self._make_linked_object("id", cloud.project_id, project_module.Project, exceptions.ProjectNotFound) + if self.cloud is None: + return None + cloud = self.cloud + if not isinstance(cloud, _base.BaseCloud): + return None + return make_linked(cloud) + diff --git a/scratchattach/_core/site/codegen_config.json b/scratchattach/_core/site/codegen_config.json new file mode 100644 index 00000000..ed506e12 --- /dev/null +++ b/scratchattach/_core/site/codegen_config.json @@ -0,0 +1,8 @@ +{ + "sync_target_directory": "../../sync_api/site", + "async_target_directory": "../../async_api/site", + "exclude": [ + "./__init__.py" + ], + "include_directories": [] +} \ No newline at end of file diff --git a/scratchattach/_core/site/comment.py b/scratchattach/_core/site/comment.py new file mode 100644 index 00000000..8e01ba66 --- /dev/null +++ b/scratchattach/_core/site/comment.py @@ -0,0 +1,228 @@ +"""Comment class""" +from __future__ import annotations + +import warnings +import html + +from dataclasses import dataclass +from typing_extensions import assert_never +from typing import Union, Optional, Any +from enum import Enum, auto + +from . import user, project, studio, session +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions + +class CommentSource(Enum): + PROJECT = auto() + USER_PROFILE = auto() + STUDIO = auto() + UNKNOWN = auto() + +@dataclass +class Comment(BaseSiteComponent): + """ + Represents a Scratch comment (on a profile, studio or project) + """ + id: Optional[int | str] = None + source: CommentSource = CommentSource.UNKNOWN + source_id: Optional[int | str] = None + cached_replies: Optional[list[Comment]] = None + parent_id: Optional[int | str] = None + cached_parent_comment: Optional[Comment] = None + commentee_id: Optional[int] = None + content: Optional[str] = None + reply_count: Optional[int] = None + written_by_scratchteam: Optional[bool] = None + author_id: Optional[int] = None + author_name: Optional[str] = None + + _session: Optional[session.Session] = None + + def __str__(self): + return self.text + + def update(self): + warnings.warn("Warning: Comment objects can't be updated") + return False # Objects of this type cannot be updated + + def _update_from_data(self, data: dict[str, str | dict | Any]): + self.id = data["id"] + self.parent_id = data.get("parent_id") + self.commentee_id = data.get("commentee_id") + self.content = str(data["content"]) + self.datetime_created = data["datetime_created"] + + author = data.get("author", {}) + self.author_name = author.get("username", self.author_name) + self.author_id = author.get("id", self.author_id) + self.written_by_scratchteam = author.get("scratchteam", self.written_by_scratchteam) + self.reply_count = data.get("reply_count", self.reply_count) + + source: str = data.get("source") + if self.source is CommentSource.UNKNOWN: + self.source = { + "project": CommentSource.PROJECT, + "studio": CommentSource.STUDIO, + "profile": CommentSource.USER_PROFILE, + None: CommentSource.UNKNOWN, + }[source] + + self.source_id = data.get("source_id", self.source_id) + + @property + def text(self) -> str: + """ + Parsed version of Comment.content. This removes any escape codes, e.g. ''' becomes ', an apostrophe + """ + if self.source is CommentSource.USER_PROFILE: + # user profile comments do not seem to be escaped + return self.content + + return str(html.unescape(self.content)) + + # Methods for getting related entities + + def author(self) -> user.User: + return self._make_linked_object("username", self.author_name, user.User, exceptions.UserNotFound) + + def place(self) -> user.User | studio.Studio | project.Project: + """ + Returns the place (the project, profile or studio) where the comment was posted as Project, User, or Studio object. + + If the place can't be traced back, None is returned. + """ + if self.source == CommentSource.USER_PROFILE: + return self._make_linked_object("username", self.source_id, user.User, exceptions.UserNotFound) + elif self.source == CommentSource.STUDIO: + return self._make_linked_object("id", self.source_id, studio.Studio, exceptions.UserNotFound) + elif self.source == CommentSource.PROJECT: + return self._make_linked_object("id", self.source_id, project.Project, exceptions.UserNotFound) + else: + assert_never(self.source) + + def parent_comment(self) -> Comment | None: + if self.parent_id is None: + return None + + if self.cached_parent_comment is not None: + return self.cached_parent_comment + + if self.source == "profile": + self.cached_parent_comment = user.User(username=self.source_id, _session=self._session).comment_by_id( + self.parent_id) + + elif self.source == "project": + p = project.Project(id=self.source_id, _session=self._session) + p.update() + self.cached_parent_comment = p.comment_by_id(self.parent_id) + + elif self.source == "studio": + self.cached_parent_comment = studio.Studio(id=self.source_id, _session=self._session).comment_by_id( + self.parent_id) + + return self.cached_parent_comment + + def replies(self, *, use_cache: bool = True, limit=40, offset=0): + """ + Keyword Arguments: + use_cache (bool): Returns the replies cached on the first reply fetch. This makes it SIGNIFICANTLY faster for profile comments. Warning: For profile comments, the replies are retrieved and cached on object creation. + """ + if (self.cached_replies is None) or (not use_cache): + if self.source == CommentSource.USER_PROFILE: + self.cached_replies = user.User(username=self.source_id, _session=self._session).comment_by_id( + self.id).cached_replies[offset:offset + limit] + + elif self.source == CommentSource.PROJECT: + p = project.Project(id=self.source_id, _session=self._session) + p.update() + self.cached_replies = p.comment_replies(comment_id=self.id, limit=limit, offset=offset) + + elif self.source == CommentSource.STUDIO: + self.cached_replies = studio.Studio(id=self.source_id, _session=self._session).comment_replies( + comment_id=self.id, limit=limit, offset=offset) + + return self.cached_replies + + # Methods for dealing with the comment + + def reply(self, content, *, commentee_id=None): + """ + Posts a reply comment to the comment. + + Warning: + Scratch only shows comments replying to top-level comments, and all replies to replies are actually replies to top-level comments in the API. + + Therefore, if this comment is a reply, this method will not reply to the comment itself but to the corresponding top-level comment. + + Args: + content (str): Comment content to post. + + Keyword args: + commentee_id (None or str): If set to None (default), it will automatically fill out the commentee ID with the user ID of the parent comment author. Set it to "" to mention no user. + + + Returns: + scratchattach.Comment: The created comment. + :param content: Content of the comment to send + :param commentee_id: ID of user to reply to + """ + + self._assert_auth() + parent_id = str(self.id) + if self.parent_id is not None: + parent_id = str(self.parent_id) + if commentee_id is None: + if self.author_id: + commentee_id = self.author_id + else: + commentee_id = "" + + if self.source == CommentSource.USER_PROFILE: + return user.User(username=self.source_id, _session=self._session).reply_comment(content, + parent_id=str(parent_id), + commentee_id=commentee_id) + if self.source == CommentSource.PROJECT: + p = project.Project(id=self.source_id, _session=self._session) + p.update() + return p.reply_comment(content, parent_id=str(parent_id), commentee_id=commentee_id) + + if self.source == CommentSource.STUDIO: + return studio.Studio(id=self.source_id, _session=self._session).reply_comment(content, + parent_id=str(parent_id), + commentee_id=commentee_id) + raise ValueError(f"Unknown source: {self.source}") + + def delete(self): + """ + Deletes the comment. + """ + self._assert_auth() + if self.source == CommentSource.USER_PROFILE: + return user.User(username=self.source_id, _session=self._session).delete_comment(comment_id=self.id) + + elif self.source == CommentSource.PROJECT: + p = project.Project(id=self.source_id, _session=self._session) + p.update() + return p.delete_comment(comment_id=self.id) + + elif self.source == CommentSource.STUDIO: + return studio.Studio(id=self.source_id, _session=self._session).delete_comment(comment_id=self.id) + + return None # raise error? + + def report(self): + """ + Reports the comment to the Scratch team. + """ + self._assert_auth() + if self.source == CommentSource.USER_PROFILE: + user.User(username=self.source_id, _session=self._session).report_comment(comment_id=self.id) + + elif self.source == CommentSource.PROJECT: + p = project.Project(id=self.source_id, _session=self._session) + p.update() + p.report_comment(comment_id=self.id) + + elif self.source == CommentSource.STUDIO: + studio.Studio(id=self.source_id, _session=self._session).report_comment(comment_id=self.id) diff --git a/scratchattach/_core/site/forum.py b/scratchattach/_core/site/forum.py new file mode 100644 index 00000000..0ddbad54 --- /dev/null +++ b/scratchattach/_core/site/forum.py @@ -0,0 +1,436 @@ +"""ForumTopic and ForumPost classes""" +from __future__ import annotations + +import warnings +from dataclasses import dataclass, field +from typing import Optional, Any +from urllib.parse import urlparse, parse_qs +import xml.etree.ElementTree as ET + +from bs4 import BeautifulSoup, Tag + +from . import user +from . import session as module_session +from scratchattach.utils.commons import headers +from scratchattach.utils import exceptions, commons +from ._base import BaseSiteComponent +from scratchattach.utils.requests import requests + +@dataclass +class ForumTopic(BaseSiteComponent): + ''' + Represents a Scratch forum topic. + + Attributes: + + :.id: + + :.title: + + :.category_name: + + :.last_updated: + + Attributes only available if the object was created using scratchattach.get_topic_list or scratchattach.Session.connect_topic_list: + + :.reply_count: + + :.view_count: + + :.update(): Updates the attributes + ''' + id: int + title: str + category_name: Optional[str] = None + last_updated: Optional[str] = None + _session: Optional[module_session.Session] = field(default=None) + reply_count: Optional[int] = field(default=None) + view_count: Optional[int] = field(default=None) + + def __str__(self): + return f"-F {self.title} ({self.id})" + + def __post_init__(self): + # Info on how the .update method has to fetch the data: + self.update_function = requests.get + self.update_api = f"https://scratch.mit.edu/discuss/feeds/topic/{self.id}/" + + # Headers and cookies: + if self._session is None: + self._headers = headers + self._cookies = {} + else: + self._headers = self._session.get_headers() + self._cookies = self._session.get_cookies() + + # Headers for operations that require accept and Content-Type fields: + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def update(self): + # As there is no JSON API for getting forum topics anymore, + # the data has to be retrieved from the XML feed. + response = self.update_function( + self.update_api, + headers = self._headers, + cookies = self._cookies, timeout=20 # fetching forums can take very long + ) + # Check for 429 error: + if "429" in str(response): + return "429" + + # Parse XML response + if response.status_code == 200: + try: + root = ET.fromstring(response.text) + namespace = {'atom': 'http://www.w3.org/2005/Atom'} + + title = root.findtext('atom:title', namespaces=namespace).replace("Latest posts on ","") + category_name = root.findall('.//atom:entry', namespaces=namespace)[0].findtext('.//atom:title', namespaces=namespace).split(" :: ")[1] + last_updated = root.findtext('atom:updated', namespaces=namespace) + + except Exception as e: + raise exceptions.ScrapeError(str(e)) + else: + raise exceptions.ForumContentNotFound + self.title = title + self.category_name = category_name + self.last_updated = last_updated + return True + + @classmethod + def from_id(cls, __id: int, session: module_session.Session, update: bool = False): + new = cls(id=__id, _session=session, title="", last_updated="", category_name="") + if update: + new.update() + return new + + def _update_from_data(self, data: dict[str, Any]): + self.__dict__.update(data) + + def posts(self, *, page=1, order="oldest") -> list[ForumPost]: + """ + Args: + page (int): The page of the forum topic that should be returned. First page is at index 1. + + Returns: + list: A list containing the posts from the specified page of the forum topic + """ + if order != "oldest": + warnings.warn("Warning: All post orders except for 'oldest' are deprecated and no longer work") # For backwards compatibility + + posts = [] + + try: + url = f"https://scratch.mit.edu/discuss/topic/{self.id}/?page={page}" + response = requests.get(url, headers=headers, cookies=self._cookies) + except Exception as e: + raise exceptions.FetchError(str(e)) + try: + soup = BeautifulSoup(response.content, 'html.parser') + soup_elm = soup.find("div", class_="djangobb") + assert isinstance(soup_elm, Tag) + try: + pagination_div = soup_elm.find('div', class_='pagination') + assert isinstance(pagination_div, Tag) + num_pages = int(pagination_div.find_all('a', class_='page')[-1].text) + except Exception: + num_pages = 1 + + try: + # get topic category: + topic_category = "" + breadcrumb_ul = soup_elm.find_all('ul')[1] # Find the second ul element + if breadcrumb_ul: + assert isinstance(breadcrumb_ul, Tag) + link = breadcrumb_ul.find_all('a')[1] # Get the right anchor tag + topic_category = link.text.strip() # Extract and strip text content + except Exception as e: + warnings.warn(f"Warning: Couldn't scrape topic category for topic {self.id} - {e}") + topic_category = "" + + # get corresponding posts: + post_htmls = soup.find_all('div', class_='blockpost') + for raw_post in post_htmls: + if not isinstance(raw_post, Tag): + continue + post = ForumPost(id=int(str(raw_post['id']).replace("p", "")), topic_id=self.id, _session=self._session, topic_category=topic_category, topic_num_pages=num_pages) + post.update_from_html(raw_post) + + posts.append(post) + except Exception as e: + raise exceptions.ScrapeError() from e + + return posts + + def first_post(self): + """ + Returns: + scratchattach.forum.ForumPost: An object representing the first topic post + """ + posts = self.posts(page=1) + if len(posts) > 0: + return posts[0] + +@dataclass +class ForumPost(BaseSiteComponent): + ''' + Represents a Scratch forum post. + + Attributes: + + :.id: + + :.author_name: The name of the user who created this post + + :.author_avatar_url: + + :.posted: The date the post was made + + :.topic_id: The id of the topic this post is in + + :.topic_name: The name of the topic the post is in + + :.topic_category: The name of the category the post topic is in + + :.topic_num_pages: The number of pages the post topic has + + :.deleted: Whether the post was deleted (always False because deleted posts can't be retrieved anymore) + + :.html_content: Returns the content as HTML + + :.content: Returns the content as text + + :.post_index: The index that the post has in the topic + + :.update(): Updates the attributes + ''' + id: int = field(default=0) + topic_id: int = field(default=0) + topic_name: str = field(default="") + topic_category: str = field(default="") + topic_num_pages: int = field(default=0) + author_name: str = field(default="") + author_avatar_url: str = field(default="") + posted: str = field(default="") + deleted: bool = field(default=False) + html_content: str = field(default="") + content: str = field(default="") + post_index: int = field(default=0) + _session: Optional[module_session.Session] = field(default=None) + def __post_init__(self): + + # A forum post can't be updated the usual way as there is no API anymore + self.update_api = "" + + # Headers and cookies: + if self._session is None: + self._headers = headers + self._cookies = {} + else: + self._headers = self._session.get_headers() + self._cookies = self._session.get_cookies() + + # Headers for operations that require accept and Content-Type fields: + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def update_function(self, *args, **kwargs): + raise TypeError("Forum posts cannot be updated like this") + + def update(self): + """ + Updates the attributes of the ForumPost object. + As there is no API for retrieving a single post anymore, this requires reloading the forum page. + """ + page = 1 + posts = ForumTopic.from_id(self.topic_id, session=self._session).posts(page=1) + while posts != []: + matching = list(filter(lambda x : int(x.id) == int(self.id), posts)) + if len(matching) > 0: + this = matching[0] + break + page += 1 + posts = ForumTopic.from_id(self.topic_id, session=self._session).posts(page=page) + else: + return False + self._update_from_data(vars(this)) + + def _update_from_data(self, data: dict[str, Any]): + self.__dict__.update(data) + return True + + def update_from_html(self, soup_html: Tag): + return self._update_from_html(soup_html) + + def _update_from_html(self, soup_html: Tag): + post_index_elm = soup_html.find('span', class_='conr') + assert isinstance(post_index_elm, Tag) + id_attr = soup_html['id'] + assert isinstance(id_attr, str) + posted_elm = soup_html.find('a', href=True) + assert isinstance(posted_elm, Tag) + content_elm = soup_html.find('div', class_='post_body_html') + assert isinstance(content_elm, Tag) + author_name_elm = soup_html.select_one('dl dt a') + assert isinstance(author_name_elm, Tag) + topic_name_elm = soup_html.find('h3') + assert isinstance(topic_name_elm, Tag) + + self.post_index = int(post_index_elm.text.strip('#')) + self.id = int(id_attr.replace("p", "")) + self.posted = posted_elm.text.strip() + self.content = content_elm.text.strip() + self.html_content = str(soup_html.find('div', class_='post_body_html')) + self.author_name = author_name_elm.text.strip() + self.author_avatar_url = str(author_name_elm['href']) + self.topic_name = topic_name_elm.text.strip() + return True + + def topic(self): + """ + Returns: + scratchattach.forum.ForumTopic: An object representing the forum topic this post is in. + """ + return self._make_linked_object("id", self.topic_id, ForumTopic, exceptions.ForumContentNotFound) + + def ocular_reactions(self): + return requests.get(f"https://my-ocular.jeffalo.net/api/reactions/{self.id}", timeout=10).json() + + def author(self): + """ + Returns: + scratchattach.user.User: An object representing the user who created this forum post. + """ + return self._make_linked_object("username", self.author_name, user.User, exceptions.UserNotFound) + + def edit(self, new_content: str): + """ + Changes the content of the forum post. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_post` or through another method that requires authentication. You must own the forum post. + + Args: + new_content (str): The text that the forum post will be set to. + """ + + self._assert_auth() + + cookies = dict(self._cookies) + cookies["accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" + cookies["Content-Type"] = "application/x-www-form-urlencoded" + + r = requests.post( + f"https://scratch.mit.edu/discuss/post/{self.id}/edit/", + headers = { + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "accept-language": "de,en;q=0.9", + "cache-control": "max-age=0", + "content-type": "application/x-www-form-urlencoded", + "sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"101\", \"Google Chrome\";v=\"101\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"Windows\"", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "same-origin", + "sec-fetch-user": "?1", + "upgrade-insecure-requests": "1", + "Referer": f"https://scratch.mit.edu/discuss/post/{self.id}/edit/", + "x-csrftoken": "a" + }, + cookies = cookies, + json = f"csrfmiddlewaretoken=a&body={new_content}&", + timeout = 10, + ) + + +def get_topic(topic_id) -> ForumTopic: + + """ + Gets a forum topic without logging in. Data received from Scratch's RSS feed XML API. + + Args: + topic_id (int): ID of the requested forum topic + + Returns: + scratchattach.forum.ForumTopic: An object representing the requested forum topic + + Warning: + Scratch's API uses very heavy caching for logged out users, therefore the returned data will not be up to date. + + Any methods that require authentication will not work on the returned object. + + If you need up-to-date data or want to use methods that require authentication, create the object with :meth:`scratchattach.session.Session.connect_topic` instead. + """ + return commons._get_object("id", topic_id, ForumTopic, exceptions.ForumContentNotFound) + + +def get_topic_list(category_id, *, page=1): + + """ + Gets the topics from a forum category without logging in. Data web-scraped from Scratch's forums UI. + + Args: + category_id (str): ID of the forum category + + Keyword Arguments: + page (str): Page of the category topics that should be returned + + Returns: + list: A list containing the forum topics from the specified category + + Warning: + Scratch's API uses very heavy caching for logged out users, therefore the returned data will not be up to date. + + Any methods that require authentication will not work on the returned objects. + + If you need up-to-date data or want to use methods that require authentication, get the forum topics with :meth:`scratchattach.session.Session.connect_topic_list` instead. + """ + + try: + response = requests.get(f"https://scratch.mit.edu/discuss/{category_id}/?page={page}") + soup = BeautifulSoup(response.content, 'html.parser') + except Exception as e: + raise exceptions.FetchError(str(e)) + + try: + category_name = soup.find('h4').find("span").get_text() + except Exception as e: + raise exceptions.BadRequest("Invalid category id") + + try: + topics = soup.find_all('tr') + topics.pop(0) + return_topics = [] + + for topic in topics: + title_link = topic.find('a') + title = title_link.text.strip() + topic_id = title_link['href'].split('/')[-2] + + columns = topic.find_all('td') + columns = [column.text for column in columns] + if len(columns) == 1: + # This is a sticky topic -> Skip it + continue + + last_updated = columns[3].split(" ")[0] + " " + columns[3].split(" ")[1] + + return_topics.append(ForumTopic(id=int(topic_id), title=title, category_name=category_name, last_updated=last_updated, reply_count=int(columns[1]), view_count=int(columns[2]))) + return return_topics + except Exception as e: + raise exceptions.ScrapeError(str(e)) + + +def youtube_link_to_scratch(link: str): + """ + Converts a YouTube url (in multiple formats) like https://youtu.be/1JTgg4WVAX8?si=fIEskaEaOIRZyTAz + to a link like https://scratch.mit.edu/discuss/youtube/1JTgg4WVAX8 + """ + url_parse = urlparse(link) + query_parse = parse_qs(url_parse.query) + if 'v' in query_parse: + video_id = query_parse['v'][0] + else: + video_id = url_parse.path.split('/')[-1] + return f"https://scratch.mit.edu/discuss/youtube/{video_id}" diff --git a/scratchattach/_core/site/project.py b/scratchattach/_core/site/project.py new file mode 100644 index 00000000..1cc0f591 --- /dev/null +++ b/scratchattach/_core/site/project.py @@ -0,0 +1,884 @@ +"""Project and PartialProject classes""" + +from __future__ import annotations + +import json +import random +import base64 +import time +import warnings +import zipfile +from io import BytesIO +from typing import Callable, Union + +from dataclasses import dataclass, field +from typing import Any, Optional +from typing_extensions import deprecated + +from scratchattach.site.typed_dicts import ProjectDict +from . import user, comment, studio, session +from scratchattach.utils import exceptions +from scratchattach.utils import commons +from scratchattach.utils.commons import empty_project_json, headers +from ._base import BaseSiteComponent + +# from scratchattach.other.project_json_capabilities import ProjectBody +from scratchattach import editor +from scratchattach.utils.requests import requests + +CREATE_PROJECT_USES: list[float] = [] + + +@dataclass +class PartialProject(BaseSiteComponent): + """ + Represents an unshared Scratch project that can't be accessed. + """ + + id: Union[str, int] = field(kw_only=True, default=0) + "The project id" + url: str = field(kw_only=True, default="") + "The project url" + title: str = field(kw_only=True, default="") + author_name: str = field(kw_only=True, default="") + "The username of the author" + comments_allowed: bool = field(kw_only=True, default=False) + "whether comments are enabled" + comment_count: int = 0 + """The number of comments on the project (this may be unreliable)""" + instructions: str = field(kw_only=True, default="") + notes: str = field(kw_only=True, default="") + "The 'Notes and Credits' section" + created: str = field(kw_only=True, default="") + "The date of the project creation" + last_modified: str = field(kw_only=True, default="") + "The date when the project was modified the last time" + share_date: str = field(kw_only=True, default="") + thumbnail_url: str = field(kw_only=True, default="") + remix_parent: Optional[Union[str, int]] = field(kw_only=True, default="") + parent_title: Optional[str] = field(kw_only=True, default=None) + remix_root: Optional[Union[str, int]] = field(kw_only=True, default="") + loves: int = field(kw_only=True, default=0) + "The project's love count" + favorites: int = field(kw_only=True, default=0) + "The project's favorite count" + remix_count: int = field(kw_only=True, default=0) + "The number of remixes" + views: int = field(kw_only=True, default=0) + "The view count" + project_token: Optional[str] = field(kw_only=True, default=None) + "The project token (required to access the project json)" + _moderation_status: Optional[str] = field(kw_only=True, default=None) + _session: Optional[session.Session] = field(kw_only=True, default=None) + + def __str__(self): + return f"Unshared project with id {self.id}" + + def __post_init__(self) -> None: + + # Info on how the .update method has to fetch the data: + self.update_function: Callable = requests.get + self.update_api = f"https://api.scratch.mit.edu/projects/{self.id}" + + # Headers and cookies: + if self._session is None: + self._headers = headers + self._cookies = {} + else: + self._headers = self._session.get_headers() + self._cookies = self._session.get_cookies() + + # Headers for operations that require accept and Content-Type fields: + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def _update_from_data(self, data: ProjectDict): + self.id = int(data.get("id", self.id)) + self.url = f"https://scratch.mit.edu/projects/{self.id}" + if author := data.get("author"): + self.author_name = author.get("username", self.author_name) + self.author_name = data.get("username", self.author_name) + self.comments_allowed = data.get("comments_allowed", self.comments_allowed) + self.instructions = data.get("instructions", self.instructions) + self.notes = data.get("description", self.notes) + + if history := data.get("history"): + self.created = history.get("created", self.created) + self.last_modified = history.get("modified", self.last_modified) + self.share_date = history.get("shared", self.share_date) + + self.thumbnail_url = data.get("image", self.thumbnail_url) + + # NOTE: if we have no value, then we set it to None instead of empty string. + # TODO: consider changing this behavior + remix_data = data.get("remix", {}) + self.remix_parent = remix_data.get("parent") + self.remix_root = remix_data.get("root") + + if stats := data.get("stats"): + self.favorites = stats.get("favorites", self.favorites) + self.loves = stats.get("loves", self.loves) + self.remix_count = stats.get("remixes", self.remix_count) + self.views = stats.get("views", self.views) + + self.title = data.get("title", self.title) + self.project_token = data.get("project_token", None) + + # the typed dict here isn't perfect: + # code as in {"code": "not found"} + # if the project is unshared, then we get that error code + return "code" not in data + + def __rich__(self): + from rich.panel import Panel + from rich.table import Table + from rich import box + from rich.markup import escape + + url = f"[link={self.url}]{self.title}[/]" + + ret = Table.grid(expand=True) + ret.add_column(ratio=1) + ret.add_column(ratio=3) + + info = Table(box=box.SIMPLE) + info.add_column(url, overflow="fold") + info.add_column(f"#{self.id}", overflow="fold") + + info.add_row("By", self.author_name) + info.add_row("Created", escape(self.created)) + info.add_row("Shared", escape(self.share_date)) + info.add_row("Modified", escape(self.last_modified)) + info.add_row("Comments allowed", escape(str(self.comments_allowed))) + info.add_row("Loves", str(self.loves)) + info.add_row("Faves", str(self.favorites)) + info.add_row("Remixes", str(self.remix_count)) + info.add_row("Views", str(self.views)) + + desc = Table(box=box.SIMPLE) + desc.add_row("Instructions", escape(self.instructions)) + desc.add_row("Notes & Credits", escape(self.notes)) + + ret.add_row(Panel(info, title=url), Panel(desc, title="Description")) + + return ret + + @property + def embed_url(self): + """ + Returns: + the url of the embed of the project + """ + return f"{self.url}/embed" + + def remixes(self, *, limit=40, offset=0) -> list[Project]: + """ + Returns: + list: A list containing the remixes of the project, each project is represented by a Project object. + """ + response = commons.api_iterative(f"https://api.scratch.mit.edu/projects/{self.id}/remixes", limit=limit, offset=offset) + return commons.parse_object_list(response, Project, self._session) + + def is_shared(self): + """ + Returns: + boolean: Returns whether the project is currently shared + """ + try: + p = get_project(self.id) + return isinstance(p, Project) + except exceptions.ProjectNotFound: + return False + + def raw_json_or_empty(self) -> dict[str, Any]: + return empty_project_json + + def create_remix(self, *, title=None, project_json=None) -> Project: # not working + """ + Creates a project on the Scratch website. + + Warning: + Don't spam this method - it WILL get you banned from Scratch. + To prevent accidental spam, a rate limit (5 projects per minute) is implemented for this function. + """ + session = self._assert_auth() + + if title is None: + if "title" in self.__dict__: + title = self.title + " remix" + else: + title = " remix" + if project_json is None: + project_json = self.raw_json_or_empty() + + if len(CREATE_PROJECT_USES) < 5: + CREATE_PROJECT_USES.insert(0, time.time()) + else: + if CREATE_PROJECT_USES[-1] < time.time() - 300: + CREATE_PROJECT_USES.pop() + else: + raise exceptions.BadRequest( + "Rate limit for remixing Scratch projects exceeded.\nThis rate limit is enforced by scratchattach, not by the Scratch API.\nFor security reasons, it cannot be turned off.\n\nDon't spam-create projects, it WILL get you banned." + ) + CREATE_PROJECT_USES.insert(0, time.time()) + + params = { + "is_remix": "1", + "original_id": self.id, + "title": title, + } + + response = requests.post( + "https://projects.scratch.mit.edu/", params=params, cookies=self._cookies, headers=self._headers, json=project_json + ).json() + _project = session.connect_project(response["content-name"]) + _project.parent_title = base64.b64decode(response["content-title"]).decode("utf-8").split(" remix")[0] + return _project + + def load_description(self): + """ + Gets the instructions of the unshared project. Requires authentication. + + Warning: + It's unclear if Scratch allows using this method. This method will create a remix of the unshared project using your account. + """ + self._assert_auth() + new_project = self.create_remix(project_json=empty_project_json) + self.instructions = new_project.instructions + self.title = new_project.parent_title or "" + + +@dataclass +class Project(PartialProject): + """ + Represents a Scratch project. + """ + + def __repr__(self): + return f"-P {self.id} ({self.title})" + + def __str__(self): + return repr(self) + + @property + def thumbnail(self) -> bytes: + with requests.no_error_handling(): + return requests.get(self.thumbnail_url).content + + def _assert_permission(self): + session = self._assert_auth() + if session.username != self.author_name: + raise exceptions.Unauthorized("You need to be authenticated as the profile owner to do this.") + + def load_description(self): + # Overrides the load_description method that exists for unshared projects + self.update() + + # -- Project contents (body/json) -- # + + def download(self, *, filename=None, dir="."): + """ + Downloads the project json to the given directory. + + Args: + filename (str): The name that will be given to the downloaded file. + dir (str): The path of the directory the file will be saved in. + """ + try: + if filename is None: + filename = str(self.id) + if not (dir.endswith("/") or dir.endswith("\\")): + dir += "/" + self.update() + response = requests.get( + f"https://projects.scratch.mit.edu/{self.id}?token={self.project_token}", + timeout=10, + ) + filename = filename.removesuffix(".sb3") + with open(f"{dir}{filename}.sb3", "wb") as f: + f.write(response.content) + except Exception as exc: + raise (exceptions.FetchError("Method only works for projects created with Scratch 3")) from exc + + @deprecated("Use raw_json instead") + def get_json(self) -> str: + """ + Downloads the project json and returns it as a string + """ + try: + self.update() + response = requests.get( + f"https://projects.scratch.mit.edu/{self.id}?token={self.project_token}", + timeout=10, + ) + return response.text + + except Exception as exc: + raise (exceptions.FetchError("Method only works for projects created with Scratch 3")) from exc + + def body(self) -> editor.Project: + """ + Method only works for project created with Scratch 3. + + Returns: + scratchattach.editor.Project: The contents of the project as editor Project object + """ + raw_json = self.raw_json() + return editor.Project.from_json(raw_json) + + def raw_json(self): + """ + Method only works for project created with Scratch 3. + + Returns: + dict: The raw project JSON as decoded Python dictionary + """ + try: + self.update() + + except Exception as e: + raise (exceptions.FetchError(f"You're not authorized for accessing {self}.\nException: {e}")) + + with requests.no_error_handling(): + resp = requests.get( + f"https://projects.scratch.mit.edu/{self.id}?token={self.project_token}", + timeout=10, + ) + + try: + return resp.json() + except json.JSONDecodeError: + # I am not aware of any cases where this will not be a zip file + # in the future, cache a projectbody object here and just return the json + # that is fetched from there to not waste existing asset data from this zip file + + with zipfile.ZipFile(BytesIO(resp.content)) as zipf: + return json.load(zipf.open("project.json")) + + def raw_json_or_empty(self): + return self.raw_json() + + def creator_agent(self): + """ + Method only works for project created with Scratch 3. + + Returns: + str: The user agent of the browser that this project was saved with. + """ + return self.raw_json()["meta"]["agent"] + + def set_body(self, project_body: editor.Project): + """ + Sets the project's contents You can use this to upload projects to the Scratch website. + Returns a dict with Scratch's raw JSON API response. + + Args: + project_body (scratchattach.ProjectBody): A ProjectBody object containing the contents of the project + """ + self._assert_permission() + + return self.set_json(project_body.to_json()) + + def set_json(self, json_data): + """ + Sets the project json. You can use this to upload projects to the Scratch website. + Returns a dict with Scratch's raw JSON API response. + + Args: + json_data (dict or JSON): The new project JSON as encoded JSON object or as dict + """ + + self._assert_permission() + + if not isinstance(json_data, dict): + json_data = json.loads(json_data) + + return requests.put( + f"https://projects.scratch.mit.edu/{self.id}", + headers=self._headers, + cookies=self._cookies, + json=json_data, + ).json() + + def upload_json_from(self, project_id: int | str): + """ + Uploads the project json from the project with the given id to the project represented by this Project object + """ + self._assert_auth() + other_project = self._session.connect_project(project_id) # type: ignore + self.set_json(other_project.raw_json()) + + # -- other -- # + + def author(self) -> user.User: + """ + Returns: + scratchattach.user.User: An object representing the Scratch user who created this project. + """ + return self._make_linked_object("username", self.author_name, user.User, exceptions.UserNotFound) + + def studios(self, *, limit=40, offset=0): + """ + Returns: + list: A list containing the studios this project is in, each studio is represented by a Studio object. + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/studios", + limit=limit, + offset=offset, + add_params=f"&cachebust={random.randint(0, 9999)}", + ) + return commons.parse_object_list(response, studio.Studio, self._session) + + def comments(self, *, limit=40, offset=0) -> list["comment.Comment"]: + """ + Returns the comments posted on the project (except for replies. To get replies use :meth:`scratchattach.project.Project.comment_replies`). + + Keyword Arguments: + page: The page of the comments that should be returned. + limit: Max. amount of returned comments. + + Returns: + list: A list containing the requested comments as Comment objects. + """ + + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/comments/", + limit=limit, + offset=offset, + add_params=f"&cachebust={random.randint(0, 9999)}", + _headers=self._headers, + cookies=self._cookies, + ) + for i in response: + i["source"] = "project" + i["source_id"] = self.id + return commons.parse_object_list(response, comment.Comment, self._session) + + def comment_replies(self, *, comment_id, limit=40, offset=0): + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/comments/{comment_id}/replies/", + limit=limit, + offset=offset, + add_params=f"&cachebust={random.randint(0, 9999)}", + _headers=self._headers, + cookies=self._cookies, + ) + for x in response: + x["parent_id"] = comment_id + x["source"] = "project" + x["source_id"] = self.id + return commons.parse_object_list(response, comment.Comment, self._session) + + def comment_by_id(self, comment_id): + """ + Returns: + scratchattach.comments.Comment: A Comment object representing the requested comment. + """ + # https://api.scratch.mit.edu/users/TimMcCool/projects/404369790/comments/439984518 + data = requests.get( + f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/comments/{comment_id}", + headers=self._headers, + cookies=self._cookies, + ).json() + + if data is None or data.get("code") == "NotFound": + raise exceptions.CommentNotFound(f"Cannot find comment #{comment_id} on -P {self.id} by -U {self.author_name}") + + _comment = comment.Comment( + id=data["id"], _session=self._session, source=comment.CommentSource.PROJECT, source_id=self.id + ) + _comment._update_from_data(data) + return _comment + + def love(self): + """ + Posts a love on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + session = self._assert_auth() + r = requests.post( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/loves/user/{session.username}", + headers=self._headers, + cookies=self._cookies, + ).json() + if "userLove" in r: + if r["userLove"] is False: + self.love() + else: + raise exceptions.APIError(str(r)) + + def unlove(self): + """ + Removes the love from this project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + session = self._assert_auth() + r = requests.delete( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/loves/user/{session.username}", + headers=self._headers, + cookies=self._cookies, + ).json() + if "userLove" in r: + if r["userLove"] is True: + self.unlove() + else: + raise exceptions.APIError(str(r)) + + def favorite(self): + """ + Posts a favorite on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + session = self._assert_auth() + r = requests.post( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/favorites/user/{session.username}", + headers=self._headers, + cookies=self._cookies, + ).json() + if "userFavorite" in r: + if r["userFavorite"] is False: + self.favorite() + else: + raise exceptions.APIError(str(r)) + + def unfavorite(self): + """ + Removes the favorite from this project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + session = self._assert_auth() + r = requests.delete( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/favorites/user/{session.username}", + headers=self._headers, + cookies=self._cookies, + ).json() + if "userFavorite" in r: + if r["userFavorite"] is True: + self.unfavorite() + else: + raise exceptions.APIError(str(r)) + + def post_view(self): + """ + Increases the project's view counter by 1. Doesn't require a login. + """ + requests.post( + f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/views/", + headers=headers, + ) + + def set_fields(self, fields_dict, *, use_site_api=False): + """ + Sets fields. By default, ueses the api.scratch.mit.edu/projects/xxx/ PUT API. + + Keyword Arguments: + use_site_api (bool): + When enabled, the fields are set using the scratch.mit.edu/site-api API. + This function allows setting more fields than Project.set_fields. + For example, you can also share / unshare the project by setting the "shared" field. + According to the Scratch team, this API is deprecated. As of 2024 it's still fully functional though. + """ + self._assert_permission() + if use_site_api: + r = requests.put( + f"https://scratch.mit.edu/site-api/projects/all/{self.id}", + headers=self._headers, + cookies=self._cookies, + json=fields_dict, + ).json() + else: + r = requests.put( + f"https://api.scratch.mit.edu/projects/{self.id}", + headers=self._headers, + cookies=self._cookies, + json=fields_dict, + ).json() + return self._update_from_data(r) + + def turn_off_commenting(self): + """ + Disables commenting on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + data = {"comments_allowed": False} + self.set_fields(data) + + def turn_on_commenting(self): + """ + Enables commenting on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + data = {"comments_allowed": True} + self.set_fields(data) + + def toggle_commenting(self): + """ + Switches commenting on / off on the project (If comments are on, they will be turned off, else they will be turned on). You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + data = {"comments_allowed": not self.comments_allowed} + self.set_fields(data) + + def share(self): + """ + Shares the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self._assert_permission() + requests.put( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/share/", + headers=self._json_headers, + cookies=self._cookies, + ) + + def unshare(self): + """ + Unshares the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self._assert_permission() + requests.put( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/unshare/", + headers=self._json_headers, + cookies=self._cookies, + ) + + ''' doesn't work. the API's response is valid (no errors), but the fields don't change + def move_to_trash(self): + """ + Moves the project to trash folder. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self.set_fields({"id":int(self.id), "visibility": "trshbyusr", "isPublished" : False}, use_site_api=True)''' + + def set_thumbnail(self, *, file): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self._assert_permission() + with open(file, "rb") as f: + thumbnail = f.read() + requests.post( + f"https://scratch.mit.edu/internalapi/project/thumbnail/{self.id}/set/", + data=thumbnail, + headers=self._headers, + cookies=self._cookies, + ) + + def delete_comment(self, *, comment_id): + """ + Deletes a comment by its ID. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + comment_id: The id of the comment that should be deleted + """ + self._assert_permission() + return requests.delete( + f"https://api.scratch.mit.edu/proxy/comments/project/{self.id}/comment/{comment_id}/", + headers=self._headers, + cookies=self._cookies, + ) + + def report_comment(self, *, comment_id): + """ + Reports a comment by its ID to the Scratch team. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + comment_id: The id of the comment that should be reported + """ + self._assert_auth() + return requests.delete( + f"https://api.scratch.mit.edu/proxy/comments/project/{self.id}/comment/{comment_id}/report", + headers=self._headers, + cookies=self._cookies, + ) + + def post_comment(self, content, *, parent_id="", commentee_id=""): + """ + Posts a comment on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + content: Content of the comment that should be posted + + Keyword Arguments: + parent_id: ID of the comment you want to reply to. If you don't want to mention a user, don't put the argument. + commentee_id: ID of the user that will be mentioned in your comment and will receive a message about your comment. If you don't want to mention a user, don't put the argument. + + Returns: + scratchattach.comments.Comment: Comment object representing the posted comment. + """ + self._assert_auth() + data = { + "commentee_id": commentee_id, + "content": str(content), + "parent_id": parent_id, + } + r = json.loads( + requests.post( + f"https://api.scratch.mit.edu/proxy/comments/project/{self.id}/", + headers=(self._json_headers | {"referer": "https://scratch.mit.edu/projects/" + str(self.id) + "/"}), + cookies=self._cookies, + data=json.dumps(data), + ).text + ) + if "id" not in r: + raise exceptions.CommentPostFailure(r) + _comment = comment.Comment(id=r["id"], _session=self._session, source=comment.CommentSource.PROJECT, source_id=self.id) + _comment._update_from_data(r) + return _comment + + def reply_comment(self, content, *, parent_id, commentee_id=""): + """ + Posts a reply to a comment on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + content: Content of the comment that should be posted + + Warning: + Only replies to top-level comments are shown on the Scratch website. Replies to replies are actually replies to the corresponding top-level comment in the API. + + Therefore, parent_id should be the comment id of a top level comment. + + Keyword Arguments: + parent_id: ID of the comment you want to reply to + commentee_id: ID of the user you are replying to + """ + return self.post_comment(content, parent_id=parent_id, commentee_id=commentee_id) + + def set_title(self, text): + """ + Changes the projects title. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self.set_fields({"title": text}) + + def set_instructions(self, text): + """ + Changes the projects instructions. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self.set_fields({"instructions": text}) + + def set_notes(self, text): + """ + Changes the projects notes and credits. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self.set_fields({"description": text}) + + @deprecated("Deprecated because ScratchDB is down indefinitely.") + def ranks(self): + """ + Gets information about the project's ranks. Fetched from ScratchDB. + + Warning: + This method is deprecated because ScratchDB is down indefinitely. + + Returns: + dict: A dict containing the project's ranks. If the ranks aren't available, all values will be -1. + """ + return requests.get(f"https://scratchdb.lefty.one/v3/project/info/{self.id}").json()["statistics"]["ranks"] + + def moderation_status(self, *, reload: bool = False): + """ + Gets information about the project's moderation status. Fetched from jeffalo's API. + + Returns: + str: The moderation status of the project. + + These moderation statuses exist: + + safe: The project was reviewed by the Scratch team and was considered safe for everyone. + + notsafe: The project was reviewed by the Scratch team and was considered not safe for everyone (nfe). It can't appear in search results, on the explore page and on the front page. + + notreviewed: The project hasn't been reviewed yet. + + no_remixes: Unable to fetch the project's moderation status. + """ + if self._moderation_status and not reload: + return self._moderation_status + + try: + return requests.get(f"https://jeffalo.net/api/nfe/?project={self.id}").json()["status"] + except Exception as exc: + raise exceptions.FetchError from exc + + def visibility(self): + """ + Returns info about the project's visibility. Requires authentication. + """ + session = self._assert_auth() + return requests.get( + f"https://api.scratch.mit.edu/users/{session.username}/projects/{self.id}/visibility", + headers=self._headers, + cookies=self._cookies, + ).json() + + +# ------ # + + +def get_project(project_id) -> Project: + """ + Gets a project without logging in. + + Args: + project_id (int): Project id of the requested project + + Returns: + scratchattach.project.Project: An object representing the requested project. + + Warning: + Any methods that require authentication (like project.love) will not work on the returned object. + + If you want to use these methods, get the project with :meth:`scratchattach.session.Session.connect_project` instead. + """ + warnings.warn( + "For methods that require authentication, use session.connect_project instead of get_project.\n" + "If you want to remove this warning, " + "use `warnings.filterwarnings('ignore', category=scratchattach.ProjectAuthenticationWarning)`.\n" + "To ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use " + "`warnings.filterwarnings('ignore', category=scratchattach.GetAuthenticationWarning)`.", + exceptions.ProjectAuthenticationWarning, + ) + return commons._get_object("id", project_id, Project, exceptions.ProjectNotFound) + + +def search_projects(*, query="", mode="trending", language="en", limit=40, offset=0): + """ + Uses the Scratch search to search projects. + + Keyword arguments: + query (str): The query that will be searched. + mode (str): Has to be one of these values: "trending", "popular" or "recent". Defaults to "trending". + language (str): A language abbreviation, defaults to "en". (Depending on the language used on the Scratch website, Scratch displays you different results.) + limit (int): Max. amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: List that contains the search results. + """ + if not query: + raise ValueError("The query can't be empty for search") + response = commons.api_iterative( + "https://api.scratch.mit.edu/search/projects", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}&q={query}", + ) + return commons.parse_object_list(response, Project) + + +def explore_projects(*, query="*", mode="trending", language="en", limit=40, offset=0): + """ + Gets projects from the explore page. + + Keyword arguments: + query (str): Specifies the tag of the explore page. To get the projects from the "All" tag, set this argument to "*". + mode (str): Has to be one of these values: "trending", "popular" or "recent". Defaults to "trending". + language (str): A language abbreviation, defaults to "en". (Depending on the language used on the Scratch website, Scratch displays you different explore pages.) + limit (int): Max. amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: List that contains the explore page projects. + """ + if not query: + raise ValueError("The query can't be empty for search") + response = commons.api_iterative( + "https://api.scratch.mit.edu/explore/projects", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}&q={query}", + ) + return commons.parse_object_list(response, Project) diff --git a/scratchattach/_core/site/session.py b/scratchattach/_core/site/session.py new file mode 100644 index 00000000..47ac8f95 --- /dev/null +++ b/scratchattach/_core/site/session.py @@ -0,0 +1,1802 @@ +"""Session class and login function""" + +from __future__ import annotations +from types import TracebackType + +import base64 +import bs4 +import datetime +import hashlib +import json +import pathlib +import random +import re +import time +import warnings +import zlib + +from dataclasses import dataclass, field +from typing import Literal, Optional, TypeVar, TYPE_CHECKING, overload, Any, Union, cast, Self +from contextlib import contextmanager +from threading import local + +from scratchattach import editor + +Type = type + +if TYPE_CHECKING: + from _typeshed import FileDescriptorOrPath, SupportsRead + from scratchattach.cloud._base import BaseCloud + + T = TypeVar("T", bound=BaseCloud) +else: + T = TypeVar("T") + +from bs4 import BeautifulSoup, Tag +from typing_extensions import deprecated + +from . import activity, classroom, forum, studio, user, project, backpack_asset, alert +from . import typed_dicts + +# noinspection PyProtectedMember +from ._base import BaseSiteComponent, api_iterative +from scratchattach.cloud import cloud, _base +from scratchattach.eventhandlers import message_events, filterbot +from scratchattach.other import other_apis +from scratchattach.utils import commons, exceptions +from scratchattach.utils.commons import ( + headers, + empty_project_json, + webscrape_count, + get_class_sort_mode, +) +from scratchattach._shared import http as shared_http +from ..primitives import http +from .browser_cookies import Browser, ANY, cookies_from_browser + +ratelimit_cache: dict[str, list[float]] = {} + + +def enforce_ratelimit(__type: str, name: str, amount: int = 5, duration: int = 60) -> None: + cache = ratelimit_cache + cache.setdefault(__type, []) + uses = cache[__type] + while uses and uses[-1] < time.time() - duration: + uses.pop() + if len(uses) < amount: + uses.insert(0, time.time()) + return + raise exceptions.RateLimitedError( + f"Rate limit for {name} exceeded.\n" + "This rate limit is enforced by scratchattach, not by the Scratch API.\n" + "For security reasons, it cannot be turned off.\n\n" + "Don't spam-create studios or similar, it WILL get you banned." + ) + + +C = TypeVar("C", bound=BaseSiteComponent) + + +class UnauthSession: + http_session: http._HTTPSession + + +@dataclass +class Session(BaseSiteComponent[typed_dicts.SessionDict]): + """ + Represents a Scratch log in / session. Stores authentication data (session id and xtoken). + + Attributes: + id: The session id associated with the login + username: The username associated with the login + xtoken: The xtoken associated with the login + email: The email address associated with the logged in account + new_scratcher: True if the associated account is a new Scratcher + mute_status: Information about commenting restrictions of the associated account + banned: Returns True if the associated account is banned + """ + + http_session: http._HTTPSession = field(repr=False, kw_only=True) + + username: str = field(repr=False, default="") + _user: Optional[user.User] = field(repr=False, default=None) + + id: str = field(repr=False, default="") + session_string: Optional[str] = field(repr=False, default=None) + xtoken: Optional[str] = field(repr=False, default=None) + email: Optional[str] = field(repr=False, default=None) + + new_scratcher: bool = field(repr=False, default=False) + mute_status: Any = field(repr=False, default=None) + banned: bool = field(repr=False, default=False) + + time_created: datetime.datetime = field( + repr=False, default=datetime.datetime.fromtimestamp(0.0) + ) + language: str = field(repr=False, default="en") + + has_outstanding_email_confirmation: bool = field(repr=False, default=False) + is_teacher: bool = field(repr=False, default=False) + is_teacher_invitee: bool = field(repr=False, default=False) + ocular_token: Optional[str] = field( + repr=False, default=None + ) # note that this is a header, not a cookie + _session: Session | UnauthSession = field(kw_only=True, init=False) + + def __str__(self) -> str: + return f"-L {self.username}" + + def __rich__(self): + from rich.panel import Panel + from rich.table import Table + from rich import box + from rich.markup import escape + + # try: + # self.update() + # except KeyError as e: + # warnings.warn(f"Ignored KeyError: {e}") + + ret = Table( + f"[link={self.connect_linked_user().url}]{escape(self.username)}[/]", + f"Created: {self.time_created}", + expand=True, + ) + + ret.add_row("Email", escape(str(self.email))) + ret.add_row("Language", escape(str(self.language))) + ret.add_row("Mute status", escape(str(self.mute_status))) + ret.add_row("New scratcher?", str(self.new_scratcher)) + ret.add_row("Banned?", str(self.banned)) + ret.add_row( + "Has outstanding email confirmation?", str(self.has_outstanding_email_confirmation) + ) + ret.add_row("Is teacher invitee?", str(self.is_teacher_invitee)) + ret.add_row("Is teacher?", str(self.is_teacher)) + + return ret + + @property + def _username(self) -> str: + return self.username + + def __post_init__(self): + # Info on how the .update method has to fetch the data: + self.update_function = shared_http.HTTPMethod.POST + self.update_api = "https://scratch.mit.edu/session" + + # Base headers and cookies of every session: + self._headers = dict(headers) + try: + self.id = json.loads(self.id) + except json.JSONDecodeError: + pass + self._cookies = { + "scratchsessionsid": self.id, + "scratchcsrftoken": "a", + "scratchlanguage": "en", + "accept": "application/json", + "Content-Type": "application/json", + } + + self._update_http_cookies_and_headers() + + if self.id: + self._process_session_id() + + self._session = self + + if "IS_ASYNC": + + async def _aenter(self) -> Self: + await self.http_session.__aenter__() + return self + + async def _aexit( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + await self.http_session.__aexit__(exc_type, exc_val, exc_tb) + + def _enter(self) -> None: + raise TypeError("Use async with instead") + + def _exit( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + # __exit__ should exist in pair with __enter__ but never executed + pass # pragma: no cover + + else: + + def _enter(self) -> Self: # type: ignore[misc] + self.http_session.__enter__() + return self + + def _exit( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self.http_session.__exit__(exc_type, exc_val, exc_tb) + + def _update_from_data(self, data: typed_dicts.SessionDict): + # Note: there are a lot more things you can get from this data dict. + # Maybe it would be a good idea to also store the dict itself? + # self.data = data + self.xtoken = data["user"]["token"] + self._headers["X-Token"] = self.xtoken + + self.has_outstanding_email_confirmation = data["flags"][ + "has_outstanding_email_confirmation" + ] + + self.email = data["user"]["email"] + + self.new_scratcher = data["permissions"]["new_scratcher"] + self.is_teacher = data["permissions"]["educator"] + self.is_teacher_invitee = data["permissions"]["educator_invitee"] + + self.mute_status = data["permissions"]["mute_status"] + + self.username = data["user"]["username"] + self.banned = data["user"]["banned"] + + if self.banned: + warnings.warn( + f"Warning: The account {self.username} you logged in to is BANNED. Some features may not work properly." + ) + if self.has_outstanding_email_confirmation: + warnings.warn( + f"Warning: The account {self.username} you logged is not email confirmed. Some features may not work properly." + ) + return True + + def _process_session_id(self): + assert self.id + + data, self.time_created = decode_session_id(self.id) + + self.username = data["username"] + # if self._user: + # self._user.username = self.username + # else: + # self._user = user.User(_session=self, username=self.username) + + # self._user.id = data["_auth_user_id"] + self.xtoken = data["token"] + self._headers["X-Token"] = self.xtoken + + # not saving the login ip because it is a security issue, and is not very helpful + + self.language = data.get("_language", "en") + # self._cookies["scratchlanguage"] = self.language + + def _assert_ocular_auth(self) -> str: + if not self.ocular_token: + raise ValueError( + f"No ocular token supplied for {self}! You can add one by using Session.set_ocular_token(YOUR_TOKEN)." + ) + return self.ocular_token + + def _update_http_cookies_and_headers(self): + self.http_session.clear_cookies() + self.http_session.update_cookies(self._cookies) + self.http_session.clear_headers() + self.http_session.update_headers(self._headers) + + def set_ocular_token(self, token: str): + self.ocular_token = token + + def connect_linked_user(self) -> user.User: + """ + Gets the user associated with the login / session. + + Warning: + The returned User object is cached. To ensure its attribute are up to date, you need to run .update() on it. + + Returns: + scratchattach.user.User: Object representing the user associated with the session. + """ + cached = hasattr(self, "_user") + if cached: + cached = self._user is not None + + if not cached: + self._user = self.connect_user(self._username) + assert self._user is not None + return self._user + + def get_linked_user(self) -> user.User: + # backwards compatibility with v1 + + # To avoid inconsistencies with "connect" and "get", this function was renamed + return self.connect_linked_user() + + async def set_country(self, country: str = "Antarctica"): + """ + Sets the profile country of the session's associated user + + Arguments: + country (str): The country to relocate to + """ + async with self.http_session.post( + "https://scratch.mit.edu/accounts/settings/", + shared_http.options().data({"country": country}).value, + # data={"country": country}, + # headers=self._headers, + # cookies=self._cookies, + ): + pass + + async def resend_email(self, password: str): + """ + Sends a request to resend a confirmation email for this session's account + + Keyword arguments: + password (str): Password associated with the session (not stored) + """ + async with self.http_session.post( + "https://scratch.mit.edu/accounts/email_change/", + shared_http.options() + .data({"email_address": await self.get_new_email_address(), "password": password}) + .value, + # data={"email_address": self.get_new_email_address(), "password": password}, + # headers=self._headers, + # cookies=self._cookies, + ): + pass + + async def get_new_email_address(self) -> str: + """ + Gets the (unconfirmed) email address that this session has requested to transfer to, if any, + otherwise the current address. + + Returns: + str: The email that this session wants to switch to + """ + async with self.http_session.get( + "https://scratch.mit.edu/accounts/email_change/", + # headers=self._headers, + # cookies=self._cookies, + ) as response: + soup = BeautifulSoup(await response.text(), "html.parser") + + email = None + for label_span in soup.find_all("span", {"class": "label"}): + if not isinstance(label_span, Tag): + continue + if not isinstance(label_span.parent, Tag): + continue + if label_span.contents[0] == "New Email Address": + return label_span.parent.contents[-1].text.strip("\n ") + + elif label_span.contents[0] == "Current Email Address": + email = label_span.parent.contents[-1].text.strip("\n ") + if email is None: + for label_span in soup.select("form#email-change span.current-email"): + email = label_span.text + assert email is not None + return email + + async def logout(self): + """ + Sends a logout request to scratch. (Might not do anything, might log out this account on other ips/sessions.) + """ + async with self.http_session.post("https://scratch.mit.edu/accounts/logout/"): + pass + + async def set_featured_data( + self, + project_id: Optional[int] | Literal[""], + project_label: Optional[int] | Literal[""] = None, + ): + """ + Sends a request to change your featured project area. + + Positional arguments: + project_id: None -> don't change; empty string -> set to latest project (this is what most accounts have); int -> set the featured project to the one with the corresponding ID. If you do not own that project, an error is raised. + project_lavel: None -> don't change; empty string -> "Featured project"; 0 -> "Featured Tutorial"; 1 -> "Work in progress"; 2 -> "Remix this!"; 3 -> "My favorite things"; 4 -> "Why I scratch" + + Returns: + list: List that contains all messages as Activity objects. + + """ + # TODO: consider using an enum here for project label and match that with user.get_featured_data + payload: dict[str, int | str] = {} + if project_label is not None: + payload["featured_project_label"] = str(project_label) + if project_id is not None: + payload["featured_project"] = project_id + + async with self.http_session.put( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", + shared_http.options().json(payload).value, + # json=payload, + # headers=self._headers, + # cookies=self._cookies, + ) as response: + data = await response.json() + if errors := data.get("errors"): + raise Exception( + f"Backend responded with error: {errors[0] if len(errors) == 1 else errors}" + ) + + return data + + @property + def ocular_headers(self) -> dict[str, str]: + return { + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36", + "referer": "https://ocular.jeffalo.net/", + "authorization": self._assert_ocular_auth(), + } + + async def get_ocular_status(self) -> typed_dicts.OcularUserDict: + # You can use sess.connect_linked_user().ocular_status() but this uses the ocular token to work out the username. + # In the case the username does not match the session, this would mismatch, and a warning could even be issued + self._assert_ocular_auth() + + async with self.http_session.get( + "https://my-ocular.jeffalo.net/auth/me", + shared_http.options() + .disregard_default_headers() + .disregard_default_cookies() + .headers(self.ocular_headers) + .value, + ) as response: + return cast(typed_dicts.OcularUserDict, await response.json()) + + async def set_ocular_status( + self, status: Optional[str] = None, color: Optional[str] = None + ) -> None: + self._assert_ocular_auth() + old = await self.get_ocular_status() + payload = {"color": color or old["color"], "status": status or old["status"]} + + async with self.http_session.put( + f"https://my-ocular.jeffalo.net/api/user/{old['name']}", + shared_http.options() + .disregard_default_headers() + .disregard_default_cookies() + .headers(self.ocular_headers) + .json(payload) + .value, + # json=payload, + # headers=self.ocular_headers, + ) as response: + assert response.json() == {"ok": "user updated"}, ( + f"Error occured on setting ocular status. auth/me response: {old}" + ) + # assert requests.put( + # f"https://my-ocular.jeffalo.net/api/user/{old['name']}", + # json=payload, + # headers=self.ocular_headers, + # ).json() == {"ok": "user updated"}, ( + # f"Error occured on setting ocular status. auth/me response: {old}" + # ) + + async def messages( + self, *, limit: int = 40, offset: int = 0, date_limit=None, filter_by=None + ) -> list[activity.Activity]: + """ + Returns the messages. + + Keyword arguments: + limit, offset, date_limit + filter_by (str or None): Can either be None (no filter), "comments", "projects", "studios" or "forums" + + Returns: + list: List that contains all messages as Activity objects. + """ + add_params = "" + if date_limit is not None: + add_params += f"&dateLimit={date_limit}" + if filter_by is not None: + add_params += f"&filter={filter_by}" + + data: list[Any] = await api_iterative( + self, + f"https://api.scratch.mit.edu/users/{self._username}/messages", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + add_params=add_params, + ) + return activity.Activity.parse_object_list(data, self) + + def admin_messages(self, *, limit=40, offset=0) -> list[dict]: + """ + Returns your messages sent by the Scratch team (alerts). + """ + return commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/messages/admin", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + ) + + async def classroom_alerts( + self, + _classroom: Optional[classroom.Classroom | int] = None, + mode: str = "Last created", + page: Optional[int] = None, + ): + """ + Load and parse admin alerts, optionally for a specific class, using https://scratch.mit.edu/site-api/classrooms/alerts/ + + Returns: + list[alert.EducatorAlert]: A list of parsed EducatorAlert objects + """ + + if isinstance(_classroom, classroom.Classroom): + _classroom = _classroom.id + + if _classroom is None: + _classroom_str = "" + else: + _classroom_str = f"{_classroom}/" + + ascsort, descsort = get_class_sort_mode(mode) + + async with self.http_session.get( + f"https://scratch.mit.edu/site-api/classrooms/alerts/{_classroom_str}", + shared_http.options() + .params({"page": page, "ascsort": ascsort, "descsort": descsort}) + .value, + # params={"page": page, "ascsort": ascsort, "descsort": descsort}, + # headers=self._headers, + # cookies=self._cookies, + ) as response: + data = await response.json() + + # data = requests.get( + # f"https://scratch.mit.edu/site-api/classrooms/alerts/{_classroom_str}", + # params={"page": page, "ascsort": ascsort, "descsort": descsort}, + # headers=self._headers, + # cookies=self._cookies, + # ).json() + + alerts = [alert.EducatorAlert.from_json(alert_data, self) for alert_data in data] + + return alerts + + def clear_messages(self): + """ + Clears all messages. + """ + return requests.post( + "https://scratch.mit.edu/site-api/messages/messages-clear/", + headers=self._headers, + cookies=self._cookies, + timeout=10, + ).text + + def message_count(self) -> int: + """ + Returns the message count. + + Returns: + int: message count + """ + return json.loads( + requests.get( + f"https://scratch.mit.edu/messages/ajax/get-message-count/", + headers=self._headers, + cookies=self._cookies, + timeout=10, + ).text + )["msg_count"] + + # Front-page-related stuff: + + def feed(self, *, limit=20, offset=0, date_limit=None) -> list[activity.Activity]: + """ + Returns the "What's happening" section (frontpage). + + Returns: + list: List that contains all "What's happening" entries as Activity objects + """ + add_params = "" + if date_limit is not None: + add_params = f"&dateLimit={date_limit}" + data = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/following/users/activity", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + add_params=add_params, + ) + return commons.parse_object_list(data, activity.Activity, self) + + def get_feed(self, *, limit=20, offset=0, date_limit=None): + # for more consistent names, this method was renamed + return self.feed( + limit=limit, offset=offset, date_limit=date_limit + ) # for backwards compatibility with v1 + + def loved_by_followed_users(self, *, limit=40, offset=0) -> list[project.Project]: + """ + Returns the "Projects loved by Scratchers I'm following" section (frontpage). + + Returns: + list: List that contains all "Projects loved by Scratchers I'm following" + entries as Project objects + """ + data = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/following/users/loves", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + ) + return commons.parse_object_list(data, project.Project, self) + + def shared_by_followed_users(self, *, limit=40, offset=0) -> list[project.Project]: + """ + Returns the "Projects by Scratchers I'm following" section (frontpage). + This section is only visible to old accounts (until ~2018). + For newer users, this method will always return an empty list. + + Returns: + list: List that contains all "Projects by Scratchers I'm following" + entries as Project objects + """ + data = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/following/users/projects", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + ) + ret = commons.parse_object_list(data, project.Project, self) + if not ret: + warnings.warn( + f"`shared_by_followed_users` got empty list `[]`. Note that this method is not supported for " + f"accounts made after 2018." + ) + return ret + + def in_followed_studios(self, *, limit=40, offset=0) -> list["project.Project"]: + """ + Returns the "Projects in studios I'm following" section (frontpage). + This section is only visible to old accounts (until ~2018) + For newer users, this method will always return an empty list. + + Returns: + list: List that contains all "Projects in studios I'm following" section" + entries as Project objects + """ + data = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/following/studios/projects", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + ) + ret = commons.parse_object_list(data, project.Project, self) + if not ret: + warnings.warn( + f"`in_followed_studios` got empty list `[]`. Note that this method is not supported for " + f"accounts made after 2018." + ) + return ret + + # -- Project JSON editing capabilities --- + # These are set to staticmethods right now, but they probably should not be + def connect_empty_project_pb(self) -> editor.Project: + pb = editor.Project.from_json( + empty_project_json + ) # in the future, ideally just init a new editor.Project, instead of loading an empty one + pb._session = self + return pb + + def connect_pb_from_dict(self, project_json: dict) -> editor.Project: + pb = editor.Project.from_json(project_json) + pb._session = self + return pb + + def connect_pb_from_file(self, path_to_file) -> editor.Project: + pb = editor.Project.from_sb3(path_to_file) + pb._session = self + return pb + + @staticmethod + def download_asset(asset_id_with_file_ext, *, filename: Optional[str] = None, fp=""): + if not (fp.endswith("/") or fp.endswith("\\")): + fp = fp + "/" + try: + if filename is None: + filename = str(asset_id_with_file_ext) + response = requests.get( + "https://assets.scratch.mit.edu/" + str(asset_id_with_file_ext), + timeout=10, + ) + open(f"{fp}{filename}", "wb").write(response.content) + except Exception: + raise (exceptions.FetchError("Failed to download asset")) + + def upload_asset(self, asset_content, *, asset_id=None, file_ext=None): + data = ( + asset_content if isinstance(asset_content, bytes) else open(asset_content, "rb").read() + ) + + if isinstance(asset_content, str): + file_ext = pathlib.Path(asset_content).suffix + file_ext = file_ext.replace(".", "") + + if asset_id is None: + asset_id = hashlib.md5(data).hexdigest() + + requests.post( + f"https://assets.scratch.mit.edu/{asset_id}.{file_ext}", + headers=self._headers, + cookies=self._cookies, + data=data, + timeout=10, + ) + + # --- Search --- + + def search_projects( + self, + *, + query: str = "", + mode: str = "trending", + language: str = "en", + limit: int = 40, + offset: int = 0, + ) -> list[project.Project]: + """ + Uses the Scratch search to search projects. + + Keyword arguments: + query (str): The query that will be searched. + mode (str): Has to be one of these values: "trending", "popular" or "recent". Defaults to "trending". + language (str): A language abbreviation, defaults to "en". + (Depending on the language used on the Scratch website, Scratch displays you different results.) + limit (int): Max. amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: List that contains the search results. + """ + query = f"&q={query}" if query else "" + + response = commons.api_iterative( + f"https://api.scratch.mit.edu/search/projects", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}{query}", + ) + return commons.parse_object_list(response, project.Project, self) + + def explore_projects( + self, + *, + query: str = "*", + mode: str = "trending", + language: str = "en", + limit: int = 40, + offset: int = 0, + ) -> list[project.Project]: + """ + Gets projects from the explore page. + + Keyword arguments: + query (str): Specifies the tag of the explore page. + To get the projects from the "All" tag, set this argument to "*". + mode (str): Has to be one of these values: "trending", "popular" or "recent". + Defaults to "trending". + language (str): A language abbreviation, defaults to "en". + (Depending on the language used on the Scratch website, Scratch displays you different explore pages.) + limit (int): Max. amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: List that contains the explore page projects. + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/explore/projects", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}&q={query}", + ) + return commons.parse_object_list(response, project.Project, self) + + def search_studios( + self, + *, + query: str = "", + mode: str = "trending", + language: str = "en", + limit: int = 40, + offset: int = 0, + ) -> list[studio.Studio]: + query = f"&q={query}" if query else "" + + response = commons.api_iterative( + f"https://api.scratch.mit.edu/search/studios", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}{query}", + ) + return commons.parse_object_list(response, studio.Studio, self) + + def explore_studios( + self, + *, + query: str = "", + mode: str = "trending", + language: str = "en", + limit: int = 40, + offset: int = 0, + ) -> list[studio.Studio]: + query = f"&q={query}" if query else "" + response = commons.api_iterative( + f"https://api.scratch.mit.edu/explore/studios", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}{query}", + ) + return commons.parse_object_list(response, studio.Studio, self) + + # --- Create project API --- + + def create_project( + self, + *, + title: Optional[str] = None, + project_json: dict = empty_project_json, + parent_id=None, + ) -> project.Project: # not working + """ + Creates a project on the Scratch website. + + Warning: + Don't spam this method - it WILL get you banned from Scratch. + To prevent accidental spam, a rate limit (5 projects per minute) is implemented for this function. + """ + enforce_ratelimit("create_scratch_project", "creating Scratch projects") + + if title is None: + title = f"Untitled-{random.randint(0, 1 << 16)}" + + params = { + "is_remix": "0" if parent_id is None else "1", + "original_id": parent_id, + "title": title, + } + + response = requests.post( + "https://projects.scratch.mit.edu/", + params=params, + cookies=self._cookies, + headers=self._headers, + json=project_json, + ).json() + return self.connect_project(response["content-name"]) + + def create_studio( + self, *, title: Optional[str] = None, description: Optional[str] = None + ) -> studio.Studio: + """ + Create a studio on the scratch website + + Warning: + Don't spam this method - it WILL get you banned from Scratch. + To prevent accidental spam, a rate limit (5 studios per minute) is implemented for this function. + """ + enforce_ratelimit("create_scratch_studio", "creating Scratch studios") + + if self.new_scratcher: + raise exceptions.Unauthorized( + f"\nNew scratchers (like {self.username}) cannot create studios." + ) + + response = requests.post( + "https://scratch.mit.edu/studios/create/", cookies=self._cookies, headers=self._headers + ) + + studio_id = webscrape_count(response.json()["redirect"], "/studios/", "/") + new_studio = self.connect_studio(studio_id) + + if title is not None: + new_studio.set_title(title) + if description is not None: + new_studio.set_description(description) + + return new_studio + + def create_class(self, title: str, desc: str = "") -> classroom.Classroom: + """ + Create a class on the scratch website + + Warning: + Don't spam this method - it WILL get you banned from Scratch. + To prevent accidental spam, a rate limit (5 classes per minute) is implemented for this function. + """ + enforce_ratelimit("create_scratch_class", "creating Scratch classes") + + if not self.is_teacher: + raise exceptions.Unauthorized(f"{self.username} is not a teacher; can't create class") + + data = requests.post( + "https://scratch.mit.edu/classes/create_classroom/", + json={"title": title, "description": desc}, + headers=self._headers, + cookies=self._cookies, + ).json() + + class_id = data[0]["id"] + return self.connect_classroom(class_id) + + # --- My stuff page --- + + def mystuff_counts(self) -> tuple[int, int, int]: + """ + Gets the number of shared projects, unshared projects, and studios as listed on the mystuff page, + and returns them in that order. + + Example usage: + shared, unshared, studios = sess.mystuff_counts() + print(f"You have {shared} shared projects, {unshared} unshared projects, and are in {studios} studios") + """ + # TODO: classrooms? + with requests.no_error_handling(): + resp = requests.get( + "https://scratch.mit.edu/mystuff/", headers=self._headers, cookies=self._cookies + ) + soup = bs4.BeautifulSoup(resp.text, "html.parser") + + shared_elem = soup.select_one("span[data-content='shared-count']") + unshared_elem = soup.select_one("span[data-content='unshared-count']") + gallery_elem = soup.select_one("span[data-content='gallery-count']") + + assert shared_elem is not None + assert unshared_elem is not None + assert gallery_elem is not None + + shared: str = shared_elem.text.strip() + unshared: str = unshared_elem.text.strip() + gallery: str = gallery_elem.text.strip() + + return int(shared), int(unshared), int(gallery) + + def mystuff_projects( + self, filter_arg: str = "all", *, page: int = 1, sort_by: str = "", descending: bool = True + ) -> list[project.Project]: + """ + Gets the projects from the "My stuff" page. + + Args: + filter_arg (str): Possible values for this parameter are "all", "shared", "unshared" and "trashed" + + Keyword Arguments: + page (int): The page of the "My Stuff" projects that should be returned + sort_by (str): The key the projects should be sorted based on. Possible values for this parameter are "" (then the projects are sorted based on last modified), "view_count", love_count", "remixers_count" (then the projects are sorted based on remix count) and "title" (then the projects are sorted based on title) + descending (boolean): Determines if the element with the highest key value (the key is specified in the sort_by argument) should be returned first. Defaults to True. + + Returns: + list: A list with the projects from the "My Stuff" page, each project is represented by a Project object. + """ + if descending: + ascsort = "" + descsort = sort_by + else: + ascsort = sort_by + descsort = "" + try: + targets = requests.get( + f"https://scratch.mit.edu/site-api/projects/{filter_arg}/?page={page}&ascsort={ascsort}&descsort={descsort}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + projects = [] + for target in targets: + projects.append( + project.Project( + id=target["pk"], + _session=self, + author_name=self._username, + comments_allowed=None, + instructions=None, + notes=None, + created=target["fields"]["datetime_created"], + last_modified=target["fields"]["datetime_modified"], + share_date=target["fields"]["datetime_shared"], + thumbnail_url="https:" + target["fields"]["thumbnail_url"], + favorites=target["fields"]["favorite_count"], + loves=target["fields"]["love_count"], + remix_count=target["fields"]["remixers_count"], + views=target["fields"]["view_count"], + title=target["fields"]["title"], + comment_count=target["fields"]["commenters_count"], + ) + ) + return projects + except Exception: + raise exceptions.FetchError() + + def mystuff_studios( + self, filter_arg: str = "all", *, page: int = 1, sort_by: str = "", descending: bool = True + ) -> list[studio.Studio]: + if descending: + ascsort = "" + descsort = sort_by + else: + ascsort = sort_by + descsort = "" + try: + params: dict[str, Union[str, int]] = { + "page": page, + "ascsort": ascsort, + "descsort": descsort, + } + targets = requests.get( + f"https://scratch.mit.edu/site-api/galleries/{filter_arg}/", + params=params, + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + studios = [] + for target in targets: + studios.append( + studio.Studio( + id=target["pk"], + _session=self, + title=target["fields"]["title"], + description=None, + host_id=target["fields"]["owner"]["pk"], + host_name=target["fields"]["owner"]["username"], + open_to_all=None, + comments_allowed=None, + image_url="https:" + target["fields"]["thumbnail_url"], + created=target["fields"]["datetime_created"], + modified=target["fields"]["datetime_modified"], + follower_count=None, + manager_count=None, + curator_count=target["fields"]["curators_count"], + project_count=target["fields"]["projecters_count"], + ) + ) + return studios + except Exception: + raise exceptions.FetchError() + + def mystuff_classes_counts(self) -> tuple[int, int]: + """ + Returns the number of open and ended classes owned by a teacher session. + If this is not a teacher session, NotATeacherError is raised + """ + with requests.no_error_handling(): + resp = requests.get( + "https://scratch.mit.edu/educators/classes/", + headers=self._headers, + cookies=self._cookies, + ) + + if resp.status_code == 403: + raise exceptions.NotATeacherError("Response 403 when getting educators/classes") + + soup = BeautifulSoup(resp.text, "html.parser") + sidebar = soup.find("div", {"id": "sidebar", "class": "tabs-index"}) + if not sidebar: + return 0, 0 + + count_elem = sidebar.find("span", {"data-content": "classroom-count"}) + ended_elem = sidebar.find("span", {"data-content": "closed-count"}) + if not count_elem or not ended_elem: + return 0, 0 + + count = str(count_elem.text).strip() + ended_count = str(ended_elem.text).strip() + + return int(count), int(ended_count) + + def mystuff_classes( + self, mode: str = "Last created", page: Optional[int] = None + ) -> list[classroom.Classroom]: + if not self.is_teacher: + self.update() + + if not self.is_teacher: + raise exceptions.Unauthorized(f"{self.username} is not a teacher; can't have classes") + ascsort, descsort = get_class_sort_mode(mode) + + classes_data = requests.get( + "https://scratch.mit.edu/site-api/classrooms/all/", + params={"page": page, "ascsort": ascsort, "descsort": descsort}, + headers=self._headers, + cookies=self._cookies, + ).json() + classes = [] + for data in classes_data: + fields = data["fields"] + educator_pf = fields["educator_profile"] + classes.append( + classroom.Classroom( + id=data["pk"], + title=fields["title"], + classtoken=fields["token"], + datetime=datetime.datetime.fromisoformat(fields["datetime_created"]), + author=user.User( + username=educator_pf["user"]["username"], + id=educator_pf["user"]["pk"], + _session=self, + ), + _session=self, + ) + ) + return classes + + def mystuff_ended_classes( + self, mode: str = "Last created", page: Optional[int] = None + ) -> list[classroom.Classroom]: + if not self.is_teacher: + raise exceptions.Unauthorized( + f"{self.username} is not a teacher; can't have (deleted) classes" + ) + ascsort, descsort = get_class_sort_mode(mode) + + classes_data = requests.get( + "https://scratch.mit.edu/site-api/classrooms/closed/", + params={"page": page, "ascsort": ascsort, "descsort": descsort}, + headers=self._headers, + cookies=self._cookies, + ).json() + classes = [] + for data in classes_data: + fields = data["fields"] + educator_pf = fields["educator_profile"] + classes.append( + classroom.Classroom( + id=data["pk"], + title=fields["title"], + classtoken=fields["token"], + datetime=datetime.datetime.fromisoformat(fields["datetime_created"]), + author=user.User( + username=educator_pf["user"]["username"], + id=educator_pf["user"]["pk"], + _session=self, + ), + _session=self, + ) + ) + return classes + + def backpack(self, limit: int = 20, offset: int = 0) -> list[backpack_asset.BackpackAsset]: + """ + Lists the assets that are in the backpack of the user associated with the session. + + Returns: + list: List that contains the backpack items + """ + data = commons.api_iterative( + f"https://backpack.scratch.mit.edu/{self._username}", + limit=limit, + offset=offset, + _headers=self._headers, + ) + return commons.parse_object_list(data, backpack_asset.BackpackAsset, self) + + def delete_from_backpack(self, backpack_asset_id) -> backpack_asset.BackpackAsset: + """ + Deletes an asset from the backpack. + + Args: + backpack_asset_id: ID of the backpack asset that will be deleted + """ + return backpack_asset.BackpackAsset(id=backpack_asset_id, _session=self).delete() + + def become_scratcher_invite(self) -> dict: + """ + If you are a new Scratcher and have been invited for becoming a Scratcher, this API endpoint will provide + more info on the invite. + """ + return requests.get( + f"https://api.scratch.mit.edu/users/{self.username}/invites", + headers=self._headers, + cookies=self._cookies, + ).json() + + # --- Connect classes inheriting from BaseCloud --- + + @overload + def connect_cloud(self, project_id, *, cloud_class: type[T]) -> T: + """ + Connects to a cloud as logged-in user. + + Args: + project_id: + + Keyword arguments: CloudClass: The class that the returned object should be of. By default, this class is + scratchattach.cloud.ScratchCloud. + + Returns: Type[scratchattach.cloud._base.BaseCloud]: An object representing the cloud of a project. Can be of any + class inheriting from BaseCloud. + """ + + @overload + def connect_cloud(self, project_id) -> cloud.ScratchCloud: + """ + Connects to a cloud (by default Scratch's cloud) as logged-in user. + + Args: + project_id: + + Keyword arguments: CloudClass: The class that the returned object should be of. By default, this class is + scratchattach.cloud.ScratchCloud. + + Returns: Type[scratchattach.cloud._base.BaseCloud]: An object representing the cloud of a project. Can be of any + class inheriting from BaseCloud. + """ + + # noinspection PyPep8Naming + def connect_cloud( + self, project_id, *, cloud_class: Optional[type[_base.BaseCloud]] = None + ) -> _base.BaseCloud: + cloud_class = cloud_class or cloud.ScratchCloud + return cloud_class(project_id=project_id, _session=self) + + def connect_scratch_cloud(self, project_id) -> cloud.ScratchCloud: + """ + Returns: + scratchattach.cloud.ScratchCloud: An object representing the Scratch cloud of a project. + """ + return cloud.ScratchCloud(project_id=project_id, _session=self) + + def connect_tw_cloud( + self, project_id, *, purpose="", contact="", cloud_host="wss://clouddata.turbowarp.org" + ) -> cloud.TwCloud: + """ + Returns: + scratchattach.cloud.TwCloud: An object representing the TurboWarp cloud of a project. + """ + return cloud.TwCloud( + project_id=project_id, + purpose=purpose, + contact=contact, + cloud_host=cloud_host, + _session=self, + ) + + # --- Connect classes inheriting from BaseSiteComponent --- + + # noinspection PyPep8Naming + # Class is camelcase here + def _make_linked_object( + self, + identificator_name, + identificator, + __class: type[C], + NotFoundException: type[Exception], + ) -> C: + """ + The Session class doesn't save the login in a ._session attribute, but IS the login ITSELF. + + Therefore, the _make_linked_object method has to be adjusted + to get it to work for in the Session class. + + Class must inherit from BaseSiteComponent + """ + # noinspection PyProtectedMember + # _get_object is protected + return commons._get_object( + identificator_name, identificator, __class, NotFoundException, self + ) + + def connect_user(self, username: str) -> user.User: + """ + Gets a user using this session, connects the session to the User object to allow authenticated actions + + Args: + username (str): Username of the requested user + + Returns: + scratchattach.user.User: An object that represents the requested user and allows you to perform actions on the user (like user.follow) + """ + return self._make_linked_object("username", username, user.User, exceptions.UserNotFound) + + @deprecated("Finding usernames by user ids has been fixed.") + def find_username_from_id(self, user_id: int) -> str: + """ + Warning: + Every time this functions is run, a comment on your profile is posted and deleted. Therefore you shouldn't run this too often. + + Returns: + str: The username that corresponds to the user id + """ + you = user.User(username=self.username, _session=self) + try: + comment = you.post_comment("scratchattach", commentee_id=int(user_id)) + except exceptions.CommentPostFailure: + raise exceptions.BadRequest( + "After posting a comment, you need to wait 10 seconds before you can connect users by id again." + ) + except exceptions.BadRequest: + raise exceptions.UserNotFound("Invalid user id") + except Exception as e: + raise e + you.delete_comment(comment_id=comment.id) + try: + username = comment.content.split('">@')[1] + username = username.split("")[0] + except IndexError: + raise exceptions.UserNotFound() + return username + + @deprecated("Finding usernames by user ids has been fixed.") + def connect_user_by_id(self, user_id: int) -> user.User: + """ + Gets a user using this session, connects the session to the User object to allow authenticated actions + + This method ... + 1) gets the username by posting a comment with the user_id as commentee_id. + 2) deletes the posted comment. + 3) fetches other information about the user using Scratch's api.scratch.mit.edu/users/username API. + + Warning: + Every time this functions is run, a comment on your profile is posted and deleted. Therefore, you shouldn't run this too often. + + Args: + user_id (int): User ID of the requested user + + Returns: + scratchattach.user.User: An object that represents the requested user and allows you to perform actions on the user (like user.follow) + """ + # noinspection PyDeprecation + return self._make_linked_object( + "username", self.find_username_from_id(user_id), user.User, exceptions.UserNotFound + ) + + def connect_project(self, project_id) -> project.Project: + """ + Gets a project using this session, connects the session to the Project object to allow authenticated actions + sess + Args: + project_id (int): ID of the requested project + + Returns: + scratchattach.project.Project: An object that represents the requested project and allows you to perform actions on the project (like project.love) + """ + return self._make_linked_object( + "id", int(project_id), project.Project, exceptions.ProjectNotFound + ) + + def connect_studio(self, studio_id) -> studio.Studio: + """ + Gets a studio using this session, connects the session to the Studio object to allow authenticated actions + + Args: + studio_id (int): ID of the requested studio + + Returns: + scratchattach.studio.Studio: An object that represents the requested studio and allows you to perform actions on the studio (like studio.follow) + """ + return self._make_linked_object( + "id", int(studio_id), studio.Studio, exceptions.StudioNotFound + ) + + def connect_classroom(self, class_id) -> classroom.Classroom: + """ + Gets a class using this session. + + Args: + class_id (str): class id of the requested class + + Returns: + scratchattach.classroom.Classroom: An object representing the requested classroom + """ + return self._make_linked_object( + "id", int(class_id), classroom.Classroom, exceptions.ClassroomNotFound + ) + + def connect_classroom_from_token(self, class_token) -> classroom.Classroom: + """ + Gets a class using this session. + + Args: + class_token (str): class token of the requested class + + Returns: + scratchattach.classroom.Classroom: An object representing the requested classroom + """ + return self._make_linked_object( + "classtoken", int(class_token), classroom.Classroom, exceptions.ClassroomNotFound + ) + + def connect_topic(self, topic_id) -> forum.ForumTopic: + """ + Gets a forum topic using this session, connects the session to the ForumTopic object to allow authenticated actions + Data is up-to-date. Data received from Scratch's RSS feed XML API. + + Args: + topic_id (int): ID of the requested forum topic (can be found in the browser URL bar) + + Returns: + scratchattach.forum.ForumTopic: An object that represents the requested forum topic + """ + return self._make_linked_object( + "id", int(topic_id), forum.ForumTopic, exceptions.ForumContentNotFound + ) + + def connect_topic_list(self, category_id, *, page=1): + """ + Gets the topics from a forum category. Data web-scraped from Scratch's forums UI. + Data is up-to-date. + + Args: + category_id (str): ID of the forum category + + Keyword Arguments: + page (str): Page of the category topics that should be returned + + Returns: + list: A list containing the forum topics from the specified category + """ + + try: + response = requests.get( + f"https://scratch.mit.edu/discuss/{category_id}/?page={page}", + headers=self._headers, + cookies=self._cookies, + ) + soup = BeautifulSoup(response.content, "html.parser") + except Exception as e: + raise exceptions.FetchError(str(e)) + + try: + category_name = soup.find("h4").find("span").get_text() + except Exception: + raise exceptions.BadRequest("Invalid category id") + + try: + topics = soup.find_all("tr") + topics.pop(0) + return_topics = [] + + for topic in topics: + title_link = topic.find("a") + title = title_link.text.strip() + topic_id = title_link["href"].split("/")[-2] + + columns = topic.find_all("td") + columns = [column.text for column in columns] + if len(columns) == 1: + # This is a sticky topic -> Skip it + continue + + last_updated = columns[3].split(" ")[0] + " " + columns[3].split(" ")[1] + + return_topics.append( + forum.ForumTopic( + _session=self, + id=int(topic_id), + title=title, + category_name=category_name, + last_updated=last_updated, + reply_count=int(columns[1]), + view_count=int(columns[2]), + ) + ) + return return_topics + except Exception as e: + raise exceptions.ScrapeError(str(e)) + + def connect_featured(self) -> other_apis.FeaturedData: + """ + Request and return connected featured projects and studios from the front page. + """ + return other_apis.get_featured_data(self) + + # --- Connect classes inheriting from BaseEventHandler --- + + def connect_message_events(self, *, update_interval=2) -> message_events.MessageEvents: + # shortcut for connect_linked_user().message_events() + return message_events.MessageEvents( + user.User(username=self.username, _session=self), update_interval=update_interval + ) + + def connect_filterbot(self, *, log_deletions=True) -> filterbot.Filterbot: + return filterbot.Filterbot( + user.User(username=self.username, _session=self), log_deletions=log_deletions + ) + + def get_session_string(self) -> str: + assert self.session_string + return self.session_string + + def get_headers(self) -> dict[str, str]: + return self._headers + + def get_cookies(self) -> dict[str, str]: + return self._cookies + + +@dataclass +class PreparedSession: + """ + Session that needs to be activated in a context manager first. Do not instantiate this yourself. + """ + + args: Any = field(repr=False) + kwargs: Any = field(repr=False) + _session: Session = field(repr=False, init=False) + + if "IS_ASYNC": + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + # __exit__ should exist in pair with __enter__ but never executed + pass # pragma: no cover + + async def __aenter__(self) -> Session: + self._session = await Session( + *self.args, **(self.kwargs | {"http_session": http._HTTPSession()}) + )._aenter() + return self._session + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + await self._session._aexit(exc_type, exc_val, exc_tb) + + else: + + def __enter__(self) -> Session: # type: ignore[misc] + self._session = Session( + *self.args, **(self.kwargs | {"http_session": http._HTTPSession()}) + )._enter() # type: ignore[func-returns-value] + return self._session + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self._session._exit(exc_type, exc_val, exc_tb) + + +# ------ # + + +def decode_session_id(session_id: str) -> tuple[dict[str, str], datetime.datetime]: + """ + Extract the JSON data from the main part of a session ID string + Session id is in the format: + :: + + p1 contains a base64 JSON string (if it starts with `.`, then it is zlib compressed) + p2 is a base 62 encoded timestamp + p3 might be a `synchronous signature` for the first 2 parts (might be useless for us) + + The dict has these attributes: + - username + - _auth_user_id + - testcookie + - _auth_user_backend + - token + - login-ip + - _language + - django_timezone + - _auth_user_hash + """ + p1, p2, _ = session_id.split(":") + p1_bytes = base64.urlsafe_b64decode(p1 + "==") + if p1.startswith('".') or p1.startswith("."): + p1_bytes = zlib.decompress(p1_bytes) + + return (json.loads(p1_bytes), datetime.datetime.fromtimestamp(commons.b62_decode(p2))) + + +# ------ # + +_global_http_session: http._HTTPSession | None = None + + +async def _get_global_http_session() -> http._HTTPSession: + global _global_http_session + if _global_http_session is None: + async with http._HTTPSession() as session: + _global_http_session = session + return _global_http_session + + +suppressed_login_warning = local() + + +@contextmanager +def suppress_login_warning(): + """ + Suppress the login warning. + """ + suppressed_login_warning.suppressed = getattr(suppressed_login_warning, "suppressed", 0) + try: + suppressed_login_warning.suppressed += 1 + yield + finally: + suppressed_login_warning.suppressed -= 1 + + +def issue_login_warning() -> None: + """ + Issue a login data warning. + """ + if getattr(suppressed_login_warning, "suppressed", 0): + return + warnings.warn( + "IMPORTANT: If you included login credentials directly in your code (e.g. session_id, session_string, ...), " + "then make sure to EITHER instead load them from environment variables or files OR remember to remove them before " + "you share your code with anyone else. If you want to remove this warning, " + "use `warnings.filterwarnings('ignore', category=scratchattach.LoginDataWarning)`", + exceptions.LoginDataWarning, + ) + + +def login_by_id( + session_id: str, *, username: Optional[str] = None, password: Optional[str] = None, xtoken=None +) -> PreparedSession: + """ + Creates a session / log in to the Scratch website with the specified session id. + Structured similarly to Session._connect_object method. + + Args: + session_id (str) + + Keyword arguments: + username (str) + password (str) + xtoken (str) + + Returns: + scratchattach.session.Session: An object that represents the created login / session + """ + # Generate session_string (a scratchattach-specific authentication method) + # should this be changed to a @property? + issue_login_warning() + if password is not None: + session_data = dict(id=session_id, username=username, password=password) + session_string = base64.b64encode(json.dumps(session_data).encode()).decode() + else: + session_string = None + + _session = PreparedSession( + (), {"id": session_id, "username": username or "", "session_string": session_string} + ) + # if xtoken is not None: + # # xtoken is retrievable from session id, so the most we can do is assert equality + # assert xtoken == _session.xtoken + + return _session + + +async def login(username, password, *, timeout: float | int = 10) -> PreparedSession: + """ + Creates a session / log in to the Scratch website with the specified username and password. + + This method ... + 1. creates a session id by posting a login request to Scratch's login API. (If this fails, scratchattach.exceptions.LoginFailure is raised) + 2. fetches the xtoken and other information by posting a request to scratch.mit.edu/session. (If this fails, a warning is displayed) + + Args: + username (str) + password (str) + + Keyword arguments: + timeout (int): Timeout for the request to Scratch's login API (in seconds). Defaults to 10. + + Returns: + scratchattach.session.Session: An object that represents the created login / session + """ + issue_login_warning() + + http_session = await _get_global_http_session() + # Post request to login API: + _headers = headers.copy() + _headers["Cookie"] = "scratchcsrftoken=a;scratchlanguage=en;" + async with http_session.post( + "https://scratch.mit.edu/login/", + shared_http.options() + .headers(_headers) + .timeout(timeout) + .json({"username": username, "password": password}) + .value, + ) as response: + try: + result = re.search('"(.*)"', response.headers["Set-Cookie"]) + assert result is not None + session_id = str(result.group()) + except Exception: + raise exceptions.LoginFailure( + "Either the provided authentication data is wrong or your network is banned from Scratch.\n\nIf you're using an online IDE (like replit.com) Scratch possibly banned its IP address. In this case, try logging in with your session id: https://github.com/TimMcCool/scratchattach/wiki#logging-in" + ) + + with suppress_login_warning(): + return login_by_id(session_id, username=username, password=password) + + +async def login_by_session_string(session_string: str) -> PreparedSession: + """ + Login using a session string. + """ + issue_login_warning() + session_string = base64.b64decode(session_string).decode() # unobfuscate + session_data = json.loads(session_string) + try: + assert session_data.get("id") + with suppress_login_warning(): + return login_by_id( + session_data["id"], + username=session_data.get("username"), + password=session_data.get("password"), + ) + except Exception: + pass + try: + assert session_data.get("session_id") + with suppress_login_warning(): + return login_by_id( + session_data["session_id"], + username=session_data.get("username"), + password=session_data.get("password"), + ) + except Exception: + pass + try: + assert session_data.get("username") and session_data.get("password") + with suppress_login_warning(): + return await login(username=session_data["username"], password=session_data["password"]) + except Exception: + pass + raise ValueError("Couldn't log in.") + + +async def login_by_io(file: SupportsRead[str]) -> PreparedSession: + """ + Login using a file object. + """ # TODO: implement async + with suppress_login_warning(): + return await login_by_session_string(file.read()) + + +async def login_by_file(file: FileDescriptorOrPath) -> PreparedSession: + """ + Login using a path to a file. + """ # TODO: implement async + with suppress_login_warning(), open(file, encoding="utf-8") as f: + return await login_by_io(f) + + +def login_from_browser(browser: Browser = ANY) -> PreparedSession: + """ + Login from a browser + """ # TODO: warn about blocking nature + cookies = cookies_from_browser(browser) + if "scratchsessionsid" in cookies: + with suppress_login_warning(): + return login_by_id(cookies["scratchsessionsid"]) + raise ValueError("Not enough data to log in.") diff --git a/scratchattach/_core/site/studio.py b/scratchattach/_core/site/studio.py new file mode 100644 index 00000000..ac5d8c2c --- /dev/null +++ b/scratchattach/_core/site/studio.py @@ -0,0 +1,676 @@ +"""Studio class""" +from __future__ import annotations + +import warnings +import json +import random + +from dataclasses import dataclass, field + +from typing_extensions import Optional + +from . import user, comment, project, activity, session +from scratchattach.site.typed_dicts import StudioDict, StudioRoleDict +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions, commons +from scratchattach.utils.commons import api_iterative, headers +from scratchattach.utils.requests import requests + + +@dataclass +class Studio(BaseSiteComponent[StudioDict]): + """ + Represents a Scratch studio. + """ + id: int + title: Optional[str] = None + description: Optional[str] = None + host_id: Optional[int] = None + "The user id of the studio host" + follower_count: Optional[int] = None + manager_count: Optional[int] = None + project_count: Optional[int] = None + image_url: Optional[str] = None + open_to_all: Optional[bool] = None + "Whether everyone is allowed to add projects" + comments_allowed: Optional[bool] = None + created: Optional[str] = None + modified: Optional[str] = None + _session: Optional[session.Session] = None + + def __post_init__(self): + # Info on how the .update method has to fetch the data: + self.update_function = requests.get + self.update_api = f"https://api.scratch.mit.edu/studios/{self.id}" + + # Headers and cookies: + if self._session is None: + self._headers = headers + self._cookies = {} + else: + self._headers = self._session._headers + self._cookies = self._session._cookies + + # Headers for operations that require accept and Content-Type fields: + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def _update_from_data(self, studio: StudioDict): + self.id = int(studio["id"]) + self.title = studio["title"] + self.description = studio["description"] + self.host_id = studio["host"] + self.open_to_all = studio["open_to_all"] + self.comments_allowed = studio["comments_allowed"] + self.image_url = studio["image"] # rename/alias to thumbnail_url? + self.created = studio["history"]["created"] + self.modified = studio["history"]["modified"] + + stats = studio.get("stats", {}) + self.follower_count = stats.get("followers", self.follower_count) + self.manager_count = stats.get("managers", self.manager_count) + self.project_count = stats.get("projects", self.project_count) + return True + + def __str__(self): + ret = f"-S {self.id}" + if self.title: + ret += f" ({self.title})" + return ret + + def __rich__(self): + from rich.panel import Panel + from rich.table import Table + from rich import box + from rich.markup import escape + + url = f"[link={self.url}]{escape(self.title)}[/]" + + ret = Table.grid(expand=True) + ret.add_column(ratio=1) + ret.add_column(ratio=3) + + info = Table(box=box.SIMPLE) + info.add_column(url, overflow="fold") + info.add_column(f"#{self.id}", overflow="fold") + info.add_row("Host ID", str(self.host_id)) + info.add_row("Followers", str(self.follower_count)) + info.add_row("Projects", str(self.project_count)) + info.add_row("Managers", str(self.manager_count)) + info.add_row("Comments allowed", str(self.comments_allowed)) + info.add_row("Open", str(self.open_to_all)) + info.add_row("Created", self.created) + info.add_row("Modified", self.modified) + + desc = Table(box=box.SIMPLE) + desc.add_row("Description", escape(self.description)) + + ret.add_row( + Panel(info, title=url), + Panel(desc, title="Description"), + ) + + return ret + + @property + def url(self): + return f"https://scratch.mit.edu/studios/{self.id}" + + @property + def thumbnail(self) -> bytes: + with requests.no_error_handling(): + return requests.get(self.image_url).content + + def follow(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/users/bookmarkers/{self.id}/add/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + + def unfollow(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/users/bookmarkers/{self.id}/remove/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + + def comments(self, *, limit=40, offset=0) -> list[comment.Comment]: + """ + Returns the comments posted on the studio (except for replies. To get replies use :meth:`scratchattach.studio.Studio.get_comment_replies`). + + Keyword Arguments: + page: The page of the comments that should be returned. + limit: Max. amount of returned comments. + + Returns: + list: A list containing the requested comments as Comment objects. + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/studios/{self.id}/comments/", limit=limit, offset=offset, add_params=f"&cachebust={random.randint(0,9999)}") + for i in response: + i["source"] = "studio" + i["source_id"] = self.id + return commons.parse_object_list(response, comment.Comment, self._session) + + def comment_replies(self, *, comment_id, limit=40, offset=0) -> list[comment.Comment]: + response = commons.api_iterative( + f"https://api.scratch.mit.edu/studios/{self.id}/comments/{comment_id}/replies", limit=limit, offset=offset, add_params=f"&cachebust={random.randint(0,9999)}") + for x in response: + x["parent_id"] = comment_id + x["source"] = "studio" + x["source_id"] = self.id + return commons.parse_object_list(response, comment.Comment, self._session) + + def comment_by_id(self, comment_id): + r = requests.get( + f"https://api.scratch.mit.edu/studios/{self.id}/comments/{comment_id}", + timeout=10, + ).json() + if r is None: + raise exceptions.CommentNotFound() + _comment = comment.Comment(id=r["id"], _session=self._session, source=comment.CommentSource.STUDIO, source_id=self.id) + _comment._update_from_data(r) + return _comment + + def post_comment(self, content, *, parent_id="", commentee_id=""): + """ + Posts a comment on the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Args: + content: Content of the comment that should be posted + + Keyword Arguments: + parent_id: ID of the comment you want to reply to. If you don't want to mention a user, don't put the argument. + commentee_id: ID of the user that will be mentioned in your comment and will receive a message about your comment. If you don't want to mention a user, don't put the argument. + + Returns: + scratchattach.comment.Comment: The posted comment as Comment object. + """ + self._assert_auth() + data = { + "commentee_id": commentee_id, + "content": str(content), + "parent_id": parent_id, + } + headers = dict(self._json_headers) + headers["referer"] = "https://scratch.mit.edu/projects/" + str(self.id) + "/" + r = requests.post( + f"https://api.scratch.mit.edu/proxy/comments/studio/{self.id}/", + headers=headers, + cookies=self._cookies, + data=json.dumps(data), + timeout=10, + ).json() + if "id" not in r: + raise exceptions.CommentPostFailure(r) + _comment = comment.Comment(id=r["id"], _session=self._session, source=comment.CommentSource.STUDIO, source_id=self.id) + _comment._update_from_data(r) + return _comment + + def delete_comment(self, *, comment_id): + # NEEDS TO BE TESTED! + """ + Deletes a comment by ID. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + comment_id: The id of the comment that should be deleted + """ + self._assert_auth() + return requests.delete( + f"https://api.scratch.mit.edu/proxy/comments/studio/{self.id}/comment/{comment_id}/", + headers=self._headers, + cookies=self._cookies, + ).headers + + def report_comment(self, *, comment_id): + # NEEDS TO BE TESTED! + """ + Reports a comment by ID to the Scratch team. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + comment_id: The id of the comment that should be reported + """ + self._assert_auth() + return requests.delete( + f"https://api.scratch.mit.edu/proxy/comments/studio/{self.id}/comment/{comment_id}/report", + headers=self._headers, + cookies=self._cookies, + ) + + def set_thumbnail(self, *, file): + """ + Sets the studio thumbnail. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Keyword Arguments: + file: The path to the image file + + Returns: + str: Scratch cdn link to the set thumbnail + """ + self._assert_auth() + with open(file, "rb") as f: + thumbnail = f.read() + + filename = file.replace("\\", "/") + if filename.endswith("/"): + filename = filename[:-1] + filename = filename.split("/").pop() + + file_type = filename.split(".").pop() + + payload1 = f'------WebKitFormBoundaryhKZwFjoxAyUTMlSh\r\nContent-Disposition: form-data; name="file"; filename="{filename}"\r\nContent-Type: image/{file_type}\r\n\r\n' + payload1 = payload1.encode("utf-8") + payload2 = b"\r\n------WebKitFormBoundaryhKZwFjoxAyUTMlSh--\r\n" + payload = b"".join([payload1, thumbnail, payload2]) + + r = requests.post( + f"https://scratch.mit.edu/site-api/galleries/all/{self.id}/", + headers={ + "accept": "*/", + "content-type": "multipart/form-data; boundary=----WebKitFormBoundaryhKZwFjoxAyUTMlSh", + "Referer": "https://scratch.mit.edu/", + "x-csrftoken": "a", + "x-requested-with": "XMLHttpRequest", + }, + data=payload, + cookies=self._cookies, + timeout=10, + ).json() + + if "errors" in r: + raise (exceptions.BadRequest(", ".join(r["errors"]))) + else: + return r["thumbnail_url"] + + def reply_comment(self, content, *, parent_id, commentee_id=""): + """ + Posts a reply to a comment on the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Args: + content: Content of the comment that should be posted + + Warning: + Only replies to top-level comments are shown on the Scratch website. Replies to replies are actually replies to the corresponding top-level comment in the API. + + Therefore, parent_id should be the comment id of a top level comment. + + Keyword Arguments: + parent_id: ID of the comment you want to reply to + commentee_id: ID of the user you are replying to + """ + self._assert_auth() + return self.post_comment( + content, parent_id=parent_id, commentee_id=commentee_id + ) + + def projects(self, limit=40, offset=0) -> list[project.Project]: + """ + Gets the studio projects. + + Keyword arguments: + limit (int): Max amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: A list containing the studio projects as Project objects + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/studios/{self.id}/projects", limit=limit, offset=offset) + return commons.parse_object_list(response, project.Project, self._session) + + def curators(self, limit=40, offset=0) -> list[user.User]: + """ + Gets the studio curators. + + Keyword arguments: + limit (int): Max amount of returned curators. + offset (int): Offset of the first returned curator. + + Returns: + list: A list containing the studio curators as User objects + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/studios/{self.id}/curators", limit=limit, offset=offset) + return commons.parse_object_list(response, user.User, self._session, "username") + + + def invite_curator(self, curator): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + try: + return requests.put( + f"https://scratch.mit.edu/site-api/users/curators-in/{self.id}/invite_curator/?usernames={curator}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + except Exception: + raise (exceptions.Unauthorized) + + def promote_curator(self, curator): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + try: + return requests.put( + f"https://scratch.mit.edu/site-api/users/curators-in/{self.id}/promote/?usernames={curator}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + except Exception: + raise (exceptions.Unauthorized) + + def remove_curator(self, curator): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + try: + return requests.put( + f"https://scratch.mit.edu/site-api/users/curators-in/{self.id}/remove/?usernames={curator}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + except Exception: + raise (exceptions.Unauthorized) + + def transfer_ownership(self, new_owner, *, password): + """ + Makes another Scratcher studio host. You need to specify your password to do this. + + Arguments: + new_owner (str): Username of new host + + Keyword arguments: + password (str): The password of your Scratch account + + Warning: + This action is irreversible! + """ + self._assert_auth() + try: + return requests.put( + f"https://api.scratch.mit.edu/studios/{self.id}/transfer/{new_owner}", + headers=self._headers, + cookies=self._cookies, + timeout=10, + json={"password":password} + ).json() + except Exception: + raise (exceptions.Unauthorized) + + + def leave(self): + """ + Removes yourself from the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + return self.remove_curator(self._session._username) + + def add_project(self, project_id): + """ + Adds a project to the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Args: + project_id: Project id of the project that should be added + """ + self._assert_auth() + return requests.post( + f"https://api.scratch.mit.edu/studios/{self.id}/project/{project_id}", + headers=self._headers, + timeout=10, + ).json() + + def remove_project(self, project_id): + """ + Removes a project from the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Args: + project_id: Project id of the project that should be removed + """ + self._assert_auth() + return requests.delete( + f"https://api.scratch.mit.edu/studios/{self.id}/project/{project_id}", + headers=self._headers, + timeout=10, + ).json() + + def managers(self, limit=40, offset=0): + """ + Gets the studio managers. + + Keyword arguments: + limit (int): Max amount of returned managers + offset (int): Offset of the first returned manager. + + Returns: + list: A list containing the studio managers as user objects + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/studios/{self.id}/managers", limit=limit, offset=offset) + return commons.parse_object_list(response, user.User, self._session, "username") + + def host(self) -> user.User: + """ + Gets the studio host. + + Returns: + scratchattach.user.User: An object representing the studio host. + """ + managers = self.managers(limit=1, offset=0) + try: + return managers[0] + except Exception: + return None + + def set_fields(self, fields_dict): + """ + Sets fields. Uses the scratch.mit.edu/site-api PUT API. + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/galleries/all/{self.id}/", + headers=headers, + cookies=self._cookies, + data=json.dumps(fields_dict), + timeout=10, + ) + + + def set_description(self, new): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self.set_fields({"description": new + "\n"}) + + def set_title(self, new): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self.set_fields({"title": new}) + + def open_projects(self): + """ + Changes the studio settings so everyone (including non-curators) is able to add projects to the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/galleries/{self.id}/mark/open/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + + def close_projects(self): + """ + Changes the studio settings so only curators can add projects to the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/galleries/{self.id}/mark/closed/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + + def turn_off_commenting(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + if self.comments_allowed: + requests.post( + f"https://scratch.mit.edu/site-api/comments/gallery/{self.id}/toggle-comments/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + self.comments_allowed = not self.comments_allowed + + def turn_on_commenting(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + if not self.comments_allowed: + requests.post( + f"https://scratch.mit.edu/site-api/comments/gallery/{self.id}/toggle-comments/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + self.comments_allowed = not self.comments_allowed + + def toggle_commenting(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.post( + f"https://scratch.mit.edu/site-api/comments/gallery/{self.id}/toggle-comments/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + self.comments_allowed = not self.comments_allowed + + def activity(self, *, limit=40, offset=0, date_limit=None): + add_params = "" + if date_limit is not None: + add_params = f"&dateLimit={date_limit}" + response = commons.api_iterative( + f"https://api.scratch.mit.edu/studios/{self.id}/activity", limit=limit, offset=offset, add_params=add_params) + return commons.parse_object_list(response, activity.Activity, self._session) + + def accept_invite(self): + self._assert_auth() + return requests.put( + f"https://scratch.mit.edu/site-api/users/curators-in/{self.id}/add/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + + def your_role(self) -> StudioRoleDict: + """ + Returns a dict with information about your role in the studio (whether you're following, curating, managing it or are invited) + """ + self._assert_auth() + return requests.get( + f"https://api.scratch.mit.edu/studios/{self.id}/users/{self._session.username}", + headers=self._headers, + cookies=self._cookies, + timeout=10, + ).json() + + def get_exact_project_count(self) -> int: + """ + Get the exact project count of a studio using a binary-search-like strategy + """ + if self.project_count is not None and self.project_count < 100: + return self.project_count + + # Get maximum possible project count before binary search + maximum = 100 + minimum = 0 + + while True: + if not self.projects(offset=maximum): + break + minimum = maximum + maximum *= 2 + + # Binary search + while True: + middle = (minimum + maximum) // 2 + projects = self.projects(limit=40, offset=middle) + + if not projects: + # too high - no projects found + maximum = middle + elif len(projects) < 40: + # we are 40 within true value, and can infer the rest + break + else: + # too low - full project list + minimum = middle + + return middle + len(projects) + + + +def get_studio(studio_id) -> Studio: + """ + Gets a studio without logging in. + + Args: + studio_id (int): Studio id of the requested studio + + Returns: + scratchattach.studio.Studio: An object representing the requested studio + + Warning: + Any methods that authentication (like studio.follow) will not work on the returned object. + + If you want to use these, get the studio with :meth:`scratchattach.session.Session.connect_studio` instead. + """ + warnings.warn( + "Warning: For methods that require authentication, use session.connect_studio instead of get_studio.\n" + "If you want to remove this warning, use warnings.filterwarnings('ignore', category=scratchattach.StudioAuthenticationWarning).\n" + "To ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use " + "`warnings.filterwarnings('ignore', category=scratchattach.GetAuthenticationWarning)`.", + exceptions.StudioAuthenticationWarning + ) + return commons._get_object("id", studio_id, Studio, exceptions.StudioNotFound) + +def search_studios(*, query="", mode="trending", language="en", limit=40, offset=0): + if not query: + raise ValueError("The query can't be empty for search") + response = commons.api_iterative( + f"https://api.scratch.mit.edu/search/studios", limit=limit, offset=offset, add_params=f"&language={language}&mode={mode}&q={query}") + return commons.parse_object_list(response, Studio) + + +def explore_studios(*, query="", mode="trending", language="en", limit=40, offset=0): + if not query: + raise ValueError("The query can't be empty for explore") + response = commons.api_iterative( + f"https://api.scratch.mit.edu/explore/studios", limit=limit, offset=offset, add_params=f"&language={language}&mode={mode}&q={query}") + return commons.parse_object_list(response, Studio) diff --git a/scratchattach/_core/site/typed_dicts.py b/scratchattach/_core/site/typed_dicts.py new file mode 100644 index 00000000..d9df928c --- /dev/null +++ b/scratchattach/_core/site/typed_dicts.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +from scratchattach.cloud import _base +from typing import TypedDict, Required, NotRequired + + +class SessionUserDict(TypedDict): + id: int + banned: bool + should_vpn: bool + username: str + token: str + thumbnailUrl: str + dateJoined: str + email: str + birthYear: int + birthMonth: int + gender: str + + +class SessionOffenseDict(TypedDict): + expiresAt: float + messageType: str + createdAt: float + + +class EmptySessionOffensesDict(TypedDict): + pass + + +class SessionOffensesDict(TypedDict): + offenses: list[SessionOffenseDict] + showWarning: bool + muteExpiresAt: float + currentMessageType: str + + +class SessionPermissionsDict(TypedDict): + admin: bool + scratcher: bool + new_scratcher: bool + invited_scratcher: bool + social: bool + educator: bool + educator_invitee: bool + student: bool + mute_status: EmptySessionOffensesDict | SessionOffensesDict + + +class SessionFlagsDict(TypedDict): + must_reset_password: bool + must_complete_registration: bool + has_outstanding_email_confirmation: bool + show_welcome: bool + confirm_email_banner: bool + unsupported_browser_banner: bool + with_parent_email: bool + project_comments_enabled: bool + gallery_comments_enabled: bool + userprofile_comments_enabled: bool + everything_is_totally_normal: bool + + +class SessionDict(TypedDict): + user: SessionUserDict + permissions: SessionPermissionsDict + flags: SessionFlagsDict + + +class OcularUserMetaDict(TypedDict): + updated: str + updatedBy: str + + +class OcularUserDict(TypedDict): + _id: str + name: str + status: str + color: str + meta: OcularUserMetaDict + + +class UserHistoryDict(TypedDict): + joined: str + + +UserProfileImagesDict = TypedDict( + "UserProfileImagesDict", {"90x90": str, "60x60": str, "55x55": str, "50x50": str, "32x32": str} +) + + +class UserProfileDict(TypedDict): + id: int + status: str + bio: str + country: str + images: UserProfileImagesDict + membership_label: NotRequired[int] + membership_avatar_badge: NotRequired[int] + + +class UserDict(TypedDict): + id: NotRequired[int] + username: NotRequired[str] + scratchteam: NotRequired[bool] + history: NotRequired[UserHistoryDict] + profile: NotRequired[UserProfileDict] + + +class CloudLogActivityDict(TypedDict): + user: str + verb: str + name: str + variable_name: NotRequired[str] + value: str | float | int + timestamp: int + cloud: _base.AnyCloud + + +class CloudActivityDict(TypedDict): + method: str + name: str + variable_name: NotRequired[str] + value: str | float | int + project_id: int + cloud: _base.AnyCloud + + +class ClassroomDict(TypedDict): + id: int + title: str + description: str + status: str + date_start: NotRequired[str] + date_end: NotRequired[str | None] + images: NotRequired[dict[str, str]] + educator: UserDict + is_closed: NotRequired[bool] + + +class StudioHistoryDict(TypedDict): + created: str + modified: str + + +class StudioStatsDict(TypedDict): + followers: int + managers: int + projects: int + + +class StudioDict(TypedDict): + id: int + title: str + description: str + host: int + open_to_all: bool + comments_allowed: bool + image: str + history: StudioHistoryDict + stats: NotRequired[StudioStatsDict] + + +class StudioRoleDict(TypedDict): + manager: bool + curator: bool + invited: bool + following: bool + + +ProjectImagesDict = TypedDict( + "ProjectImagesDict", + {"282x218": str, "216x163": str, "200x200": str, "144x108": str, "135x102": str, "100x80": str}, +) + + +class ProjectHistoryDict(TypedDict): + created: str + modified: str + shared: str + + +class ProjectStatsDict(TypedDict): + views: int + loves: int + favorites: int + remixes: int + + +class ProjectRemixDict(TypedDict): + parent: int | None + root: int | None + + +class ProjectDict(TypedDict): + id: int + title: str + description: str + instructions: str + visibility: str + public: bool + comments_allowed: bool + is_published: bool + author: UserDict + image: str + images: ProjectImagesDict + history: ProjectHistoryDict + stats: ProjectStatsDict + remix: ProjectRemixDict + project_token: str + + +class PlaceholderProjectDataMetadataDict(TypedDict): + title: str + description: str + + +# https://github.com/GarboMuffin/placeholder/blob/e1e98953342a40abbd626a111f621711f74e783b/src/routes/projects/%5Bproject%5D/%2Bpage.server.ts#L19 +class PlaceholderProjectDataDict(TypedDict): + metadata: PlaceholderProjectDataMetadataDict + md5extsToSha256: dict[str, str] + adminOwnershipToken: str | None diff --git a/scratchattach/_core/site/user.py b/scratchattach/_core/site/user.py new file mode 100644 index 00000000..6b1e2cd8 --- /dev/null +++ b/scratchattach/_core/site/user.py @@ -0,0 +1,1250 @@ +"""User class""" + +from __future__ import annotations + +import json +import random +import re +import string +import warnings +from typing import Union, cast, Optional, TypedDict +from dataclasses import dataclass, field +from datetime import datetime, timezone +from enum import Enum + +from typing_extensions import deprecated +from bs4 import BeautifulSoup, Tag + +from ._base import BaseSiteComponent +from scratchattach.eventhandlers import message_events + +from scratchattach.utils import commons +from scratchattach.utils import exceptions +from scratchattach.utils.commons import headers +from scratchattach.utils.requests import requests + +from . import project +from . import studio +from . import forum +from . import comment +from . import activity +from . import classroom +from . import typed_dicts +from . import session + + +class Rank(Enum): + """ + Possible ranks in scratch + """ + + NEW_SCRATCHER = 0 + SCRATCHER = 1 + SCRATCH_TEAM = 2 + + +class _OcularStatusMeta(TypedDict): + updated: str + updatedBy: str + + +class _OcularStatus(TypedDict): + _id: str + name: str + status: str + color: str + meta: _OcularStatusMeta + + +class Verificator: + def __init__(self, user: User, project_id: int): + self.project = user._make_linked_object("id", project_id, project.Project, exceptions.ProjectNotFound) + self.projecturl = self.project.url + self.code = "".join(random.choices(string.ascii_letters + string.digits, k=8)) + self.username = user.username + + def check(self) -> bool: + return bool( + list( + filter( + lambda x: ( + x.author_name == self.username + and (x.content == self.code or x.content.startswith(self.code) or x.content.endswith(self.code)) + ), + self.project.comments(), + ) + ) + ) + + +@dataclass +class User(BaseSiteComponent[typed_dicts.UserDict]): + """ + Represents a Scratch user. + + Attributes: + + :.join_date: + + :.about_me: + + :.wiwo: Returns the user's 'What I'm working on' section + + :.country: Returns the country from the user profile + + :.icon_url: Returns the link to the user's pfp (90x90) + + :.id: Returns the id of the user + + :.scratchteam: Retuns True if the user is in the Scratch team + + :.update(): Updates the attributes + """ + + username: str = field(kw_only=True, default="") + join_date: str = field(kw_only=True, default="") + about_me: str = field(kw_only=True, default="") + wiwo: str = field(kw_only=True, default="") + country: str = field(kw_only=True, default="") + icon_url: str = field(kw_only=True, default="") + id: int = field(kw_only=True, default=0) + scratchteam: bool = field(kw_only=True, repr=False, default=False) + is_member: bool = field(kw_only=True, repr=False, default=False) + has_ears: bool = field(kw_only=True, repr=False, default=False) + _classroom: tuple[bool, Optional[classroom.Classroom]] = field(init=False, default=(False, None)) + _headers: dict[str, str] = field(init=False, default_factory=headers.copy) + _cookies: dict[str, str] = field(init=False, default_factory=dict) + _json_headers: dict[str, str] = field(init=False, default_factory=dict) + _session: Optional[session.Session] = field(kw_only=True, default=None) + + def __str__(self): + return f"-U {self.username}" + + @property + def status(self) -> str: + return self.wiwo + + @property + def bio(self) -> str: + return self.about_me + + @property + def icon(self) -> bytes: + with requests.no_error_handling(): + return requests.get(self.icon_url).content + + @property + def name(self) -> str: + return self.username + + def __post_init__(self): + + # Info on how the .update method has to fetch the data: + self.update_function = requests.get + self.update_api = f"https://api.scratch.mit.edu/users/{self.username}" + + # cache value for classroom getter method (using @property) + # first value is whether the cache has actually been set (because it can be None), second is the value itself + # self._classroom + + # Headers and cookies: + if self._session is not None: + self._headers = self._session.get_headers() + self._cookies = self._session.get_cookies() + + # Headers for operations that require accept and Content-Type fields: + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def _update_from_data(self, data: Union[dict, typed_dicts.UserDict]): + data = cast(typed_dicts.UserDict, data) + + self.id = data.get("id", self.id) + self.username = data.get("username", self.username) + self.scratchteam = data.get("scratchteam", self.scratchteam) + if history := data.get("history"): + self.join_date = history["joined"] + + if profile := data.get("profile"): + self.about_me = profile["bio"] + self.wiwo = profile["status"] + self.country = profile["country"] + self.icon_url = profile["images"]["90x90"] + self.is_member = bool(profile.get("membership_label", False)) + self.has_ears = bool(profile.get("membership_avatar_badge", False)) + return True + + def _assert_permission(self): + self._assert_auth() + if self._session.username != self.username: + raise exceptions.Unauthorized("You need to be authenticated as the profile owner to do this.") + + @property + def url(self): + return f"https://scratch.mit.edu/users/{self.username}" + + def __rich__(self): + from rich.panel import Panel + from rich.table import Table + from rich import box + from rich.markup import escape + + featured_data = self.featured_data() or {} + + ocular_data = {} + # FIXME: ocular is down right now, so this is disabled + # ocular_data = self.ocular_status() + ocular = "No ocular status" + + if status := ocular_data.get("status"): + color_str = "" + color_data = ocular_data.get("color") + if color_data is not None: + color_str = f"[{color_data}] ⬤ [/]" + + ocular = f"[i]{escape(status)}[/]{color_str}" + + _classroom = self.classroom + url = f"[link={self.url}]{escape(self.username)}[/]" + + info = Table(box=box.SIMPLE) + info.add_column(url, overflow="fold") + info.add_column(f"#{self.id}", overflow="fold") + + info.add_row("Joined", escape(self.join_date)) + info.add_row("Country", escape(self.country)) + info.add_row("Messages", str(self.message_count())) + info.add_row("Class", str(_classroom.title if _classroom is not None else "None")) + + desc = Table("Profile", ocular, box=box.SIMPLE) + desc.add_row("About me", escape(self.about_me)) + desc.add_row("Wiwo", escape(self.wiwo)) + desc.add_row( + escape(featured_data.get("label", "Featured Project")), + escape(str(self.connect_featured_project())), + ) + + ret = Table.grid(expand=True) + + ret.add_column(ratio=1) + ret.add_column(ratio=3) + ret.add_row(Panel(info, title=url), Panel(desc, title="Description")) + + return ret + + def connect_featured_project(self) -> Optional[project.Project]: + data = self.featured_data() or {} + if pid := data.get("id"): + return self._session.connect_project(int(pid)) + if projs := self.projects(limit=1): + return projs[0] + return None + + @property + def classroom(self) -> classroom.Classroom | None: + """ + Get a user's associated classroom, and return it as a `scratchattach.classroom.Classroom` object. + If there is no associated classroom, returns `None` + """ + if not self._classroom[0]: + with requests.no_error_handling(): + resp = requests.get(f"https://scratch.mit.edu/users/{self.username}/") + soup = BeautifulSoup(resp.text, "html.parser") + + details = soup.find("p", {"class": "profile-details"}) + if details is None: + # No details, e.g. if the user is banned + return None + + assert isinstance(details, Tag) + + class_name, class_id, is_closed = None, None, False + for a in details.find_all("a"): + if not isinstance(a, Tag): + continue + href = str(a.get("href")) + if re.match(r"/classes/\d*/", href): + class_name = a.text.strip()[len("Student of: ") :] + is_closed = bool(re.search(r"\n *\(ended\)", class_name)) # as this has a \n, we can be sure + if is_closed: + class_name = re.sub(r"\n *\(ended\)", "", class_name).strip() + + class_id = int(href.split("/")[2]) + break + + if class_name: + self._classroom = ( + True, + classroom.Classroom( + _session=self._session, + id=class_id or 0, + title=class_name, + is_closed=is_closed, + ), + ) + else: + self._classroom = True, None + + return self._classroom[1] + + def does_exist(self) -> Optional[bool]: + """ + Returns: + boolean : True if the user exists, False if the user is deleted, None if an error occured + """ + with requests.no_error_handling(): + status_code = requests.get(f"https://scratch.mit.edu/users/{self.username}/").status_code + if status_code == 200: + return True + elif status_code == 404: + return False + + return None + + # Will maybe be deprecated later, but for now still has its own purpose. + # @deprecated("This function is partially deprecated. Use user.rank() instead.") + def is_new_scratcher(self): + """ + Returns: + boolean : True if the user has the New Scratcher status, else False + """ + try: + with requests.no_error_handling(): + res = requests.get(f"https://scratch.mit.edu/users/{self.username}/").text + group = res[res.rindex('') :][:70] + return "new scratcher" in group.lower() + + except Exception as e: + warnings.warn(f"Caught exception {e=}") + return None + + def message_count(self): + return json.loads( + requests.get( + f"https://api.scratch.mit.edu/users/{self.username}/messages/count/?cachebust={random.randint(0, 10000)}", + headers={ + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.3c6 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36", + }, + ).text + )["count"] + + def featured_data(self): + """ + Returns: + dict: Gets info on the user's featured project and featured label (like "Featured project", "My favorite things", etc.) + """ + try: + response = requests.get(f"https://scratch.mit.edu/site-api/users/all/{self.username}/").json() + return { + "label": response["featured_project_label_name"], + "project": dict( + id=str(response["featured_project_data"]["id"]), + author=response["featured_project_data"]["creator"], + thumbnail_url="https://" + response["featured_project_data"]["thumbnail_url"][2:], + title=response["featured_project_data"]["title"], + ), + } + except Exception: + return None + + def unfollowers(self) -> list[User]: + """ + Get all unfollowers by comparing API response and HTML response. + NOTE: This method can take a long time to run. + + Based on https://juegostrower.github.io/unfollowers/ + """ + follower_count = self.follower_count() + + # regular followers + usernames = [] + for i in range(1, 2 + follower_count // 60): + with requests.no_error_handling(): + resp = requests.get( + f"https://scratch.mit.edu/users/{self.username}/followers/", + params={"page": i}, + ) + soup = BeautifulSoup(resp.text, "html.parser") + usernames.extend(span.text.strip() for span in soup.select("span.title")) + + # api response contains all-time followers, including deleted and unfollowed + unfollowers = [] + for offset in range(0, follower_count, 40): + unfollowers.extend(user for user in self.followers(offset=offset, limit=40) if user.username not in usernames) + + return unfollowers + + def unfollower_usernames(self) -> list[str]: + return [user.username for user in self.unfollowers()] + + def follower_count(self): + with requests.no_error_handling(): + text = requests.get( + f"https://scratch.mit.edu/users/{self.username}/followers/", + headers=self._headers, + ).text + return commons.webscrape_count(text, "Followers (", ")") + + def following_count(self): + with requests.no_error_handling(): + text = requests.get( + f"https://scratch.mit.edu/users/{self.username}/following/", + headers=self._headers, + ).text + return commons.webscrape_count(text, "Following (", ")") + + def followers(self, *, limit=40, offset=0): + """ + Returns: + list: The user's followers as list of scratchattach.user.User objects + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/followers/", + limit=limit, + offset=offset, + ) + return commons.parse_object_list(response, User, self._session, "username") + + def follower_names(self, *, limit=40, offset=0): + """ + Returns: + list: The usernames of the user's followers + """ + return [i.name for i in self.followers(limit=limit, offset=offset)] + + def following(self, *, limit=40, offset=0): + """ + Returns: + list: The users that the user is following as list of scratchattach.user.User objects + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/following/", + limit=limit, + offset=offset, + ) + return commons.parse_object_list(response, User, self._session, "username") + + def following_names(self, *, limit=40, offset=0): + """ + Returns: + list: The usernames of the users the user is following + """ + return [i.name for i in self.following(limit=limit, offset=offset)] + + def is_following(self, user: str): + """ + Returns: + boolean: Whether the user is following the user provided as argument + """ + offset = 0 + following = False + + while True: + try: + following_names = self.following_names(limit=20, offset=offset) + if user in following_names: + following = True + break + if not following_names: + break + offset += 20 + except Exception as e: + print(f"Warning: API error when performing following check: {e=}") + return following + return following + + def is_followed_by(self, user): + """ + Returns: + boolean: Whether the user is followed by the user provided as argument + """ + offset = 0 + followed = False + + while True: + try: + followed_names = self.follower_names(limit=20, offset=offset) + if user in followed_names: + followed = True + break + if not followed_names: + break + offset += 20 + except Exception as e: + print(f"Warning: API error when performing following check: {e=}") + return followed + return followed + + def is_followed_by_me(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Returns: + boolean: Whether the user is followed by the user currently logged in. + """ + self._assert_auth() + with requests.no_error_handling(): + resp = requests.get( + f"https://scratch.mit.edu/users/{self.username}/", + headers=self._headers, + cookies=self._cookies, + ) + soup = BeautifulSoup(resp.text, "html.parser") + follow_btn = soup.select_one("div.follow-button") + if not follow_btn: + print("Warning: follow button not found in page.") + return False # defualt to not followed + data_control = follow_btn.get("data-control") + return data_control == "unfollow" # True if unfollow, False if not + + def project_count(self): + with requests.no_error_handling(): + text = requests.get( + f"https://scratch.mit.edu/users/{self.username}/projects/", + headers=self._headers, + ).text + return commons.webscrape_count(text, "Shared Projects (", ")") + + def studio_count(self): + with requests.no_error_handling(): + text = requests.get( + f"https://scratch.mit.edu/users/{self.username}/studios/", + headers=self._headers, + ).text + return commons.webscrape_count(text, "Studios I Curate (", ")") + + def studios_following_count(self): + with requests.no_error_handling(): + text = requests.get( + f"https://scratch.mit.edu/users/{self.username}/studios_following/", + headers=self._headers, + ).text + return commons.webscrape_count(text, "Studios I Follow (", ")") + + def studios(self, *, limit=40, offset=0) -> list[studio.Studio]: + _studios = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/studios/curate", + limit=limit, + offset=offset, + ) + studios = [] + for studio_dict in _studios: + _studio = studio.Studio(_session=self._session, id=studio_dict["id"]) + _studio._update_from_data(studio_dict) + studios.append(_studio) + return studios + + def studios_following(self) -> list[studio.Studio]: + with requests.no_error_handling(): + resp = requests.get( + f"https://scratch.mit.edu/users/{self.username}/studios_following/", + headers=self._headers, + ) + soup = BeautifulSoup(resp.text, "html.parser") + grid = soup.select_one(".media-grid") + assert grid is not None + + studios: list[studio.Studio] = [] + for studio_elem in grid.select("li.gallery.thumb.item"): + title_span = studio_elem.select_one("span.title") + assert title_span is not None + anchor = title_span.find("a") + assert anchor is not None + + href = str(anchor["href"]) + sid = int(href.split("/")[-2]) + title: str = anchor.text + + if "\n" in title: + # we do this instead of title.strip because it *could* be possible for + # a studio title to have spaces around it. But I am not 100% sure + title = title.split("\n")[0] + + studios.append(studio.Studio(id=sid, title=title, _session=self._session)) + return studios + + def projects(self, *, limit=40, offset=0) -> list[project.Project]: + """ + Returns: + list: The user's shared projects + """ + _projects = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/projects/", + limit=limit, + offset=offset, + _headers=self._headers, + ) + for p in _projects: + p["author"] = {"username": self.username} + return commons.parse_object_list(_projects, project.Project, self._session) + + def loves(self, *, limit=40, offset=0, get_full_project: bool = False) -> list[project.Project]: + """ + Returns: + list: The user's loved projects + """ + # We need to use beautifulsoup webscraping so we cant use the api_iterative function + if offset < 0: + raise exceptions.BadRequest("offset parameter must be >= 0") + if limit < 0: + raise exceptions.BadRequest("limit parameter must be >= 0") + + # There are 40 projects on display per page + # So the first page you need to view is 1 + offset // 40 + # (You have to add one because the first page is idx 1 instead of 0) + + # The final project to view is at idx offset + limit - 1 + # (You have to -1 because the index starts at 0) + # So the page number for this is 1 + (offset + limit - 1) // 40 + + # But this is a range so we have to add another 1 for the second argument + pages = range(1 + offset // 40, 2 + (offset + limit - 1) // 40) + _projects = [] + + for page in pages: + # The index of the first project on page #n is just (n-1) * 40 + first_idx = (page - 1) * 40 + + with requests.no_error_handling(): + page_content = requests.get( + f"https://scratch.mit.edu/projects/all/{self.username}/loves/?page={page}", + headers=self._headers, + ).content + + soup = BeautifulSoup(page_content, "html.parser") + + # We need to check if we are out of bounds + # If we are, we can jump out early + # This is detectable if Scratch gives you a '404' + + # We can't just detect if the 404 text is within the whole of the page content + # because it would break if someone made a project with that name + + # This page only uses

tags for the 404 text, so we can just use a soup for those + h1_tag = soup.find("h1") + if h1_tag is not None: + # Just to confirm that it's a 404, in case I am wrong. It can't hurt + if "Whoops! Our server is Scratch'ing its head" in h1_tag.text: + break + + # Each project element is a list item with the class name 'project thumb item' so we can just use that + for i, project_element in enumerate(soup.find_all("li", {"class": "project thumb item"})): + # Remember we only want certain projects: + # The current project idx = first_idx + i + # We want to start at {offset} and end at {offset + limit} + + # So the offset <= current project idx <= offset + limit + if offset <= first_idx + i <= offset + limit: + # Each of these elements provides: + # A project id + # A thumbnail link (no need to webscrape this) + # A title + # An Author (called an owner for some reason) + assert isinstance(project_element, Tag) + project_anchors = project_element.find_all("a") + # Each list item has three tags, the first two linking the project + # 1st contains tag + # 2nd contains project title + # 3rd links to the author & contains their username + + # This function is pretty handy! + # I'll use it for an id from a string like: /projects/1070616180/ + first_anchor = project_anchors[0] + second_anchor = project_anchors[1] + third_anchor = project_anchors[2] + assert isinstance(first_anchor, Tag) + assert isinstance(second_anchor, Tag) + assert isinstance(third_anchor, Tag) + project_id = commons.webscrape_count(first_anchor.attrs["href"], "/projects/", "/") + title = second_anchor.text + author = third_anchor.contents[0] + + # Instantiating a project with the properties that we know + # This may cause issues (see below) + _project = project.Project( + id=project_id, + _session=self._session, + title=title, + author_name=author, + url=f"https://scratch.mit.edu/projects/{project_id}/", + ) + if get_full_project: + # Put this under an if statement since making api requests for every single + # project will cause the function to take a lot longer + _project.update() + + _projects.append(_project) + + return _projects + + def loves_count(self): + with requests.no_error_handling(): + text = requests.get( + f"https://scratch.mit.edu/projects/all/{self.username}/loves/", + headers=self._headers, + ).text + + # If there are no loved projects, then Scratch doesn't actually display the number - so we have to catch this + soup = BeautifulSoup(text, "html.parser") + + if not soup.find("li", {"class": "project thumb item"}): + # There are no projects, so there are no projects loved + return 0 + + return commons.webscrape_count(text, "»\n\n (", ")") + + def favorites(self, *, limit=40, offset=0): + """ + Returns: + list: The user's favorite projects + """ + _projects = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/favorites/", + limit=limit, + offset=offset, + _headers=self._headers, + ) + return commons.parse_object_list(_projects, project.Project, self._session) + + def favorites_count(self): + with requests.no_error_handling(): + text = requests.get( + f"https://scratch.mit.edu/users/{self.username}/favorites/", + headers=self._headers, + ).text + return commons.webscrape_count(text, "Favorites (", ")") + + def has_badge(self) -> bool: + """ + Returns: + bool: Whether the user has a scratch membership badge on their profile (located next to the follow button) + """ + with requests.no_error_handling(): + resp = requests.get(self.url) + soup = BeautifulSoup(resp.text, "html.parser") + head = soup.find("div", {"class": "box-head"}) + if not head: + return False + for child in head.children: + if child.name == "img": + if "membership-badge.svg" in child["src"]: + return True + return False + + def toggle_commenting(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_permission() + requests.post( + f"https://scratch.mit.edu/site-api/comments/user/{self.username}/toggle-comments/", + headers=headers, + cookies=self._cookies, + ) + + def viewed_projects(self, limit=24, offset=0): + """ + Returns: + list: The user's recently viewed projects + + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_permission() + _projects = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/projects/recentlyviewed", + limit=limit, + offset=offset, + _headers=self._headers, + ) + return commons.parse_object_list(_projects, project.Project, self._session) + + def set_pfp(self, image: bytes): + """ + Sets the user's profile picture. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + # Teachers can set pfp! - Should update this method to check for that + # self._assert_permission() + requests.post( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", + headers=self._headers, + cookies=self._cookies, + files={"file": image}, + ) + + def set_bio(self, text): + """ + Sets the user's "About me" section. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + # Teachers can set bio! - Should update this method to check for that + # self._assert_permission() + requests.put( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", + headers=self._json_headers, + cookies=self._cookies, + json={"bio": text}, + ) + + def set_wiwo(self, text): + """ + Sets the user's "What I'm working on" section. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + # Teachers can also change your wiwo + # self._assert_permission() + requests.put( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", + headers=self._json_headers, + cookies=self._cookies, + json={"status": text}, + ) + + def set_featured(self, project_id, *, label=""): + """ + Sets the user's featured project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Args: + project_id: Project id of the project that should be set as featured + + Keyword Args: + label: The label that should appear above the featured project on the user's profile (Like "Featured project", "Featured tutorial", "My favorite things", etc.) + """ + self._assert_permission() + requests.put( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", + headers=self._json_headers, + cookies=self._cookies, + json={"featured_project": int(project_id), "featured_project_label": label}, + ) + + def set_forum_signature(self, text): + """ + Sets the user's discuss forum signature. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_permission() + headers = { + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "content-type": "application/x-www-form-urlencoded", + "origin": "https://scratch.mit.edu", + "referer": "https://scratch.mit.edu/discuss/settings/TimMcCool/", + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", + } + data = { + "csrfmiddlewaretoken": "a", + "signature": text, + "update": "", + } + response = requests.post( + f"https://scratch.mit.edu/discuss/settings/{self.username}/", + cookies=self._cookies, + headers=headers, + data=data, + ) + + def post_comment(self, content, *, parent_id="", commentee_id=""): + """ + Posts a comment on the user's profile. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Args: + :param content: Content of the comment that should be posted + + Keyword Arguments: + :param commentee_id: ID of the comment you want to reply to. If you don't want to mention a user, don't put the argument. + :param parent_id: ID of the user that will be mentioned in your comment and will receive a message about your comment. If you don't want to mention a user, don't put the argument. + + Returns: + scratchattach.comment.Comment: An object representing the created comment. + """ + self._assert_auth() + data = { + "commentee_id": commentee_id, + "content": str(content), + "parent_id": parent_id, + } + r = requests.post( + f"https://scratch.mit.edu/site-api/comments/user/{self.username}/add/", + headers=headers, + cookies=self._cookies, + data=json.dumps(data), + ) + if r.status_code != 200: + if "Looks like we are having issues with our servers!" in r.text: + raise exceptions.BadRequest("Invalid arguments passed") + else: + raise exceptions.CommentPostFailure(r.text) + + text = r.text + try: + data = { + "id": text.split('
')[1].split("
")[0].strip(), + "reply_count": 0, + "cached_replies": [], + } + _comment = comment.Comment( + source=comment.CommentSource.USER_PROFILE, + parent_id=None if parent_id == "" else parent_id, + commentee_id=commentee_id, + source_id=self.username, + id=data["id"], + _session=self._session, + datetime=datetime.now(), + ) + _comment._update_from_data(data) + return _comment + except Exception as e: + if '{"error": "isFlood"}' in text: + raise ( + exceptions.CommentPostFailure( + "You are being rate-limited for running this operation too often. Implement a cooldown of about 10 seconds." + ) + ) from e + elif '")[0] + error_data = json.loads(raw_error_data) + expires = error_data["mute_status"]["muteExpiresAt"] + expires = datetime.fromtimestamp(expires, timezone.utc) + raise (exceptions.CommentPostFailure(f"You have been muted. Mute expires on {expires}")) from e + else: + raise (exceptions.FetchError(f"Couldn't parse API response: {r.text!r}")) from e + + def reply_comment(self, content, *, parent_id, commentee_id=""): + """ + Replies to a comment given by its id + + Warning: + Only replies to top-level comments are shown on the Scratch website. Replies to replies are actually replies to the corresponding top-level comment in the API. + + Therefore, parent_id should be the comment id of a top level comment. + + Args: + :param content: Content of the comment that should be posted + + Keyword Arguments: + :param parent_id: ID of the comment you want to reply to + :param commentee_id: ID of the user that will be mentioned in your comment and will receive a message about your comment. If you don't want to mention a user, don't put the argument. + """ + return self.post_comment(content, parent_id=parent_id, commentee_id=commentee_id) + + def activity(self, *, limit=1000): + """ + Returns: + list: The user's activity data as parsed list of scratchattach.activity.Activity objects + """ + with requests.no_error_handling(): + soup = BeautifulSoup( + requests.get(f"https://scratch.mit.edu/messages/ajax/user-activity/?user={self.username}&max={limit}").text, + "html.parser", + ) + + activities = [] + source = soup.find_all("li") + + for data in source: + _activity = activity.Activity(_session=self._session, raw=data) + _activity._update_from_html(data) + activities.append(_activity) + + return activities + + def activity_html(self, *, limit=1000): + """ + Returns: + str: The raw user activity HTML data + """ + with requests.no_error_handling(): + return requests.get(f"https://scratch.mit.edu/messages/ajax/user-activity/?user={self.username}&max={limit}").text + + def follow(self): + """ + Follows the user represented by the User object. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/users/followers/{self.username}/add/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + ) + + def unfollow(self): + """ + Unfollows the user represented by the User object. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/users/followers/{self.username}/remove/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + ) + + def delete_comment(self, *, comment_id): + """ + Deletes a comment by its ID. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Args: + comment_id: The id of the comment that should be deleted + """ + self._assert_permission() + with requests.no_error_handling(): + return requests.post( + f"https://scratch.mit.edu/site-api/comments/user/{self.username}/del/", + headers=headers, + cookies=self._cookies, + data=json.dumps({"id": str(comment_id)}), + ) + + def report_comment(self, *, comment_id): + """ + Reports a comment by its ID to the Scratch team. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Args: + comment_id: The id of the comment that should be reported + """ + self._assert_auth() + return requests.post( + f"https://scratch.mit.edu/site-api/comments/user/{self.username}/rep/", + headers=headers, + cookies=self._cookies, + data=json.dumps({"id": str(comment_id)}), + ) + + def comments(self, *, page=1) -> list[comment.Comment] | None: + """ + Returns the comments posted on the user's profile (with replies). + + Keyword Arguments: + page: The page of the comments that should be returned. + + Returns: + list: A list containing the requested comments as Comment objects. + """ + data = [] + + with requests.no_error_handling(): + resp = requests.get(f"https://scratch.mit.edu/site-api/comments/user/{self.username}/?page={page}") + + if resp.status_code == 404: + # Profile comments seem to end at page 67, and afterwards give 404. + # Usually when page > 67. It is possible to have empty pages before + # page 67, but still have pages with content afterwards. Hence we want + # to differentiate between empty pages with a page that actually marks + # a definite end + # + # A way to reasonably reliably detect the end of the comments is by keeping track + # of how many blank pages you have seen, and breaking if you e.g. have 3 in a row: + # https://github.com/TimMcCool/scratchattach/issues/582#issuecomment-4318675630 + return None + + soup = BeautifulSoup(resp.content, "html.parser") + + _comments = soup.find_all("li", {"class": "top-level-reply"}) + for entity in _comments: + comment_id = entity.find("div", {"class": "comment"})["data-comment-id"] + user = entity.find("a", {"id": "comment-user"})["data-comment-user"] + content = str(entity.find("div", {"class": "content"}).text).strip() + time = entity.find("span", {"class": "time"})["title"] + + main_comment = { + "id": comment_id, + "author": {"username": user}, + "content": content, + "datetime_created": time, + } + _comment = comment.Comment( + source=comment.CommentSource.USER_PROFILE, + source_id=self.username, + _session=self._session, + ) + _comment._update_from_data(main_comment) + + reply_objs = [] + replies = entity.find_all("li", {"class": "reply"}) + for reply in replies: + r_comment_id = reply.find("div", {"class": "comment"})["data-comment-id"] + r_user = reply.find("a", {"id": "comment-user"})["data-comment-user"] + r_content = ( + str(reply.find("div", {"class": "content"}).text) + .strip() + .replace("\n", "") + .replace(" ", " ") + ) + r_time = reply.find("span", {"class": "time"})["title"] + reply_data = { + "id": r_comment_id, + "author": {"username": r_user}, + "content": r_content, + "datetime_created": r_time, + "parent_id": comment_id, + "cached_parent_comment": _comment, + } + _r_comment = comment.Comment( + source=comment.CommentSource.USER_PROFILE, + source_id=self.username, + _session=self._session, + cached_parent_comment=_comment, + ) + _r_comment._update_from_data(reply_data) + reply_objs.append(_r_comment) + + _comment.reply_count = len(reply_objs) + _comment.cached_replies = list(reply_objs) + + data.append(_comment) + return data + + def comment_by_id(self, comment_id) -> comment.Comment: + """ + Gets a comment on this user's profile by id. + + Warning: + For comments very far down on the user's profile, this method will take a while to find the comment. Very old comment are deleted from Scratch's database and may not appear. + + Returns: + scratchattach.comments.Comment: The request comment. + """ + + page = 1 + page_content = self.comments(page=page) + while page_content != []: + results = list(filter(lambda x: str(x.id) == str(comment_id), page_content)) + if results == []: + results = list( + filter( + lambda x: str(x.id) == str(comment_id), + [item for x in page_content for item in x.cached_replies], + ) + ) + if results != []: + return results[0] + else: + return results[0] + page += 1 + page_content = self.comments(page=page) + raise exceptions.CommentNotFound() + + def message_events(self): + return message_events.MessageEvents(self) + + @deprecated("This method is deprecated because ScratchDB is down indefinitely.") + def stats(self): + """ + Gets information about the user's stats. Fetched from ScratchDB. + + Warning: + ScratchDB is down indefinitely, therefore this method is deprecated. + + Returns: + dict: A dict containing the user's stats. If the stats aren't available, all values will be -1. + """ + try: + stats = requests.get(f"https://scratchdb.lefty.one/v3/user/info/{self.username}").json()["statistics"] + stats.pop("ranks") + except Exception: + stats = { + "loves": -1, + "favorites": -1, + "comments": -1, + "views": -1, + "followers": -1, + "following": -1, + } + return stats + + @deprecated("Warning: ScratchDB is down indefinitely, therefore this method is deprecated.") + def ranks(self): + """ + Gets information about the user's ranks. Fetched from ScratchDB. + + Warning: + ScratchDB is down indefinitely, therefore this method is deprecated. + + Returns: + dict: A dict containing the user's ranks. If the ranks aren't available, all values will be -1. + """ + try: + return requests.get(f"https://scratchdb.lefty.one/v3/user/info/{self.username}").json()["statistics"]["ranks"] + except Exception: + return { + "country": { + "loves": 0, + "favorites": 0, + "comments": 0, + "views": 0, + "followers": 0, + "following": 0, + }, + "loves": 0, + "favorites": 0, + "comments": 0, + "views": 0, + "followers": 0, + "following": 0, + } + + def ocular_status(self) -> _OcularStatus: + """ + Gets information about the user's ocular status. Ocular is a website developed by jeffalo: https://ocular.jeffalo.net/ + + Returns: + dict + """ + return requests.get(f"https://my-ocular.jeffalo.net/api/user/{self.username}").json() + + def verify_identity(self, *, verification_project_id=395330233): + """ + Can be used in applications to verify a user's identity. + + This function returns a Verifactor object. Attributs of this object: + :.projecturl: The link to the project where the user has to go to verify + :.project: The project where the user has to go to verify as scratchattach.Project object + :.code: The code that the user has to comment + + To check if the user verified successfully, call the .check() function on the returned object. + It will return True if the user commented the code. + """ + + v = Verificator(self, verification_project_id) + return v + + def rank(self) -> Rank: + """ + Finds the rank of the user. + Returns a member of the Rank enum: either Rank.NEW_SCRATCHER, Rank.SCRATCHER, or Rank.SCRATCH_TEAM. + May replace user.scratchteam and user.is_new_scratcher in the future. + """ + + if self.is_new_scratcher(): + return Rank.NEW_SCRATCHER + + if not self.scratchteam: + return Rank.SCRATCHER + + return Rank.SCRATCH_TEAM + + +# ------ # + + +def get_user(username) -> User: + """ + Gets a user without logging in. + + Args: + username (str): Username of the requested user + + Returns: + scratchattach.user.User: An object representing the requested user + + Warning: + Any methods that require authentication (like user.follow) will not work on the returned object. + + If you want to use these, get the user with :meth:`scratchattach.session.Session.connect_user` instead. + """ + warnings.warn( + "Warning: For methods that require authentication, use session.connect_user instead of get_user.\n" + "To ignore this warning, use warnings.filterwarnings('ignore', category=scratchattach.UserAuthenticationWarning).\n" + "To ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use " + "`warnings.filterwarnings('ignore', category=scratchattach.GetAuthenticationWarning)`.", + exceptions.UserAuthenticationWarning, + ) + return commons._get_object("username", username, User, exceptions.UserNotFound) diff --git a/scratchattach/_shared/http.py b/scratchattach/_shared/http.py new file mode 100644 index 00000000..125aa0ca --- /dev/null +++ b/scratchattach/_shared/http.py @@ -0,0 +1,104 @@ +from __future__ import annotations +from dataclasses import dataclass +from typing import TypeAlias, TypeVar, Protocol, runtime_checkable, Any, BinaryIO, Self, Final +from collections.abc import Iterable +from enum import Enum, auto + +_CT_co = TypeVar("_CT_co", covariant=True) + +_KT_co = TypeVar("_KT_co", covariant=True) +_VT_co = TypeVar("_VT_co", covariant=True) + + +@runtime_checkable +class SupportsItems(Protocol[_KT_co, _VT_co]): + def items(self) -> Iterable[tuple[_KT_co, _VT_co]]: ... + + +_ParamsValue: TypeAlias = str | int | float | Iterable[str | int | float] | None + +_FilesValue: TypeAlias = BinaryIO | bytes + +_JsonEmptySentinel = object() + + +def options() -> HTTPOptionsBuilder: + return HTTPOptionsBuilder() + + +class HTTPOptionsBuilder: + __slots__ = "options" + value: HTTPOptions + + def __init__(self): + self.value = HTTPOptions() + + def params( + self, + value: Iterable[tuple[str, _ParamsValue]] | SupportsItems[str, _ParamsValue] | str | None, + ) -> Self: + self.value.params = value + return self + + def content(self, value: str | bytes | None) -> Self: + self.value.content = value + return self + + def data(self, value: Iterable[tuple[str, Any]] | SupportsItems[str, Any] | None) -> Self: + self.value.data = value + return self + + def files( + self, value: Iterable[tuple[str, _FilesValue]] | SupportsItems[str, _FilesValue] | None + ) -> Self: + self.value.files = value + return self + + def json(self, value: Any) -> Self: + self.value.json = value + return self + + def headers(self, value: Iterable[tuple[str, str]] | SupportsItems[str, str] | None) -> Self: + self.value.headers = value + return self + + def disregard_default_headers(self, value: bool = True) -> Self: + self.value.disregard_default_headers = value + return self + + def cookies(self, value: Iterable[tuple[str, str]] | SupportsItems[str, str] | None) -> Self: + self.value.cookies = value + return self + + def disregard_default_cookies(self, value: bool = True) -> Self: + self.value.disregard_default_cookies = value + return self + + def timeout(self, value: int | float | None) -> Self: + self.value.timeout = value + return self + + +class HTTPOptions: + params: Iterable[tuple[str, _ParamsValue]] | SupportsItems[str, _ParamsValue] | str | None = ( + None + ) + content: str | bytes | None = None + data: Iterable[tuple[str, Any]] | SupportsItems[str, Any] | None = None + files: Iterable[tuple[str, _FilesValue]] | SupportsItems[str, _FilesValue] | None = None + json: Any = _JsonEmptySentinel + headers: Iterable[tuple[str, str]] | SupportsItems[str, str] | None = None + disregard_default_headers: bool = False + cookies: Iterable[tuple[str, str]] | SupportsItems[str, str] | None = None + disregard_default_cookies: bool = False + timeout: int | float | None = None + + +class HTTPMethod(Enum): + GET = auto() + POST = auto() + PUT = auto() + DELETE = auto() + + +_EMPTY_OPTIONS: Final = HTTPOptions() diff --git a/scratchattach/async_api/__init__.py b/scratchattach/async_api/__init__.py new file mode 100644 index 00000000..a4e88681 --- /dev/null +++ b/scratchattach/async_api/__init__.py @@ -0,0 +1,17 @@ +import httpx + +client = httpx.AsyncClient() + + +async def get_home() -> str: + resp = await client.get("https://scratch.mit.edu") + return resp.text + + +if __name__ == "__main__": + import asyncio + + async def main(): + print(await get_home()) + + asyncio.run(main()) diff --git a/scratchattach/async_api/primitives/http.py b/scratchattach/async_api/primitives/http.py new file mode 100644 index 00000000..8d22ffe9 --- /dev/null +++ b/scratchattach/async_api/primitives/http.py @@ -0,0 +1,236 @@ +from __future__ import annotations +from types import TracebackType +from collections.abc import Iterable, Mapping +from typing import Optional, Self, cast, Any, Sequence, SupportsInt, BinaryIO, TYPE_CHECKING + +if TYPE_CHECKING: + from _typeshed import SupportsKeysAndGetItem +from scratchattach._shared import http as shared_http +import contextlib +import aiohttp + +HTTPOptions = shared_http.HTTPOptions + + +class _HTTPResponse: + _async_response: aiohttp.ClientResponse + + async def text(self) -> str: + return await self._async_response.text() + + async def content(self) -> bytes: + return await self._async_response.content.read() + + async def json(self) -> Any: + return await self._async_response.json() + + @property + def headers(self) -> Mapping[str, str]: + """ + Headers are case-insensitive. + """ + return self._async_response.headers + + def get_all_headers_for_key(self, key: str) -> list[str]: + return self._async_response.headers.getall(key) + + @property + def status_code(self) -> int: + return self._async_response.status + + +class _WrappedHTTPResponse: + _aiohttp_response_context_manager: aiohttp.client._BaseRequestContextManager[aiohttp.ClientResponse] + + def __init__(self, aiohttp_response_context_manager: aiohttp.client._BaseRequestContextManager[aiohttp.ClientResponse]): + self._aiohttp_response_context_manager = aiohttp_response_context_manager + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] + ) -> None: + pass + + async def __aenter__(self) -> _HTTPResponse: + http_response = _HTTPResponse() + http_response._async_response = await self._aiohttp_response_context_manager.__aenter__() + return http_response + + async def __aexit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, tb: TracebackType | None): + await self._aiohttp_response_context_manager.__aexit__(exc_type, exc, tb) + + +class _HTTPSession: + def add_cookie(self, key: str, value: str): + self._cookies[key] = value + + def get_cookie(self, key: str) -> Optional[str]: + return self._cookies.get(key) + + def remove_cookie(self, key: str): + del self._cookies[key] + + def clear_cookies(self): + self._cookies.clear() + + def update_cookies(self, new: "SupportsKeysAndGetItem[str, str]"): + self._cookies.update(new) + + def list_cookies(self) -> Iterable[tuple[str, str]]: + return self._cookies.items() + + def add_header(self, key: str, value: str): + self._headers[key] = value + + def get_header(self, key: str) -> Optional[str]: + return self._headers.get(key) + + def remove_header(self, key: str): + del self._headers[key] + + def clear_headers(self): + self._headers.clear() + + def update_headers(self, new: "SupportsKeysAndGetItem[str, str]"): + self._headers.update(new) + + def list_headers(self) -> Iterable[tuple[str, str]]: + return self._headers.items() + + _cookies: dict[str, str] + _headers: dict[str, str] + _http_session: aiohttp.ClientSession + + def __init__(self): + self._cookies = {} + self._headers = {} + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] + ) -> None: + pass + + async def __aenter__(self) -> Self: + self._http_session = aiohttp.ClientSession(cookie_jar=aiohttp.DummyCookieJar()) + await self._http_session.__aenter__() + return self + + async def __aexit__( + self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] + ): + await self._http_session.__aexit__(exc_type, exc_val, exc_tb) + + @staticmethod + def _get_params_kwarg( + params: Any, + ) -> ( + None + | str + | Mapping[str, Sequence[str | SupportsInt] | SupportsInt] + | Sequence[tuple[str, Sequence[str | SupportsInt] | SupportsInt]] + ): + if isinstance(params, shared_http.SupportsItems): + params = params.items() + if not isinstance(params, str) and isinstance(params, Iterable): + new_params: Any = [] + for key, value in cast(Iterable[tuple[str, Iterable[str | SupportsInt] | SupportsInt | None]], params): + if not isinstance(value, str) and isinstance(value, Iterable): + for item in value: + new_params.append((key, cast(str | SupportsInt, item))) + elif value is None: + pass + else: + new_params.append((key, cast(str | SupportsInt, value))) + params = new_params + return params + + @staticmethod + def _get_data_and_files_kwargs( + data: Iterable[tuple[str, Any]] | shared_http.SupportsItems[str, Any] | None, + files: Iterable[tuple[str, BinaryIO | bytes]] | shared_http.SupportsItems[str, BinaryIO | bytes] | None, + ) -> aiohttp.FormData: + form_data = aiohttp.FormData() + if data is not None: + if isinstance(data, shared_http.SupportsItems): + data = cast(Any, data.items()) + for key, value in cast(Iterable[tuple[str, Any]], data): + form_data.add_field(key, str(value)) + if files is not None: + if isinstance(files, shared_http.SupportsItems): + files = cast(Any, files.items()) + for key, file_data in cast(Iterable[tuple[str, BinaryIO | bytes]], files): + form_data.add_field(key, file_data, filename=key) + return form_data + + @staticmethod + def _get_headers_kwarg( + default_headers: dict[str, str], headers: Iterable[tuple[str, str]] | shared_http.SupportsItems[str, str] | None + ) -> dict[str, str]: + if headers is None: + return default_headers + if isinstance(headers, shared_http.SupportsItems): + headers = cast(Iterable[tuple[str, str]], headers.items()) + return default_headers | dict(headers) + + @staticmethod + def _get_cookies_kwarg( + default_cookies: dict[str, str], cookies: Iterable[tuple[str, str]] | shared_http.SupportsItems[str, str] | None + ) -> dict[str, str]: + if cookies is None: + return default_cookies + if isinstance(cookies, shared_http.SupportsItems): + cookies = cast(Iterable[tuple[str, str]], cookies.items()) + return default_cookies | dict(cookies) + + def _get_kwargs(self, options: HTTPOptions) -> aiohttp.client._RequestOptions: # noqa: C901 + kwargs: aiohttp.client._RequestOptions = {} + if options.params is not None: + kwargs["params"] = self._get_params_kwarg(options.params) + if options.content is not None and options.data is not None: + raise ValueError('Cannot specify both "content" and "data"') + if options.content is not None and options.files is not None: + raise ValueError('Cannot specify both "content" and "files"') + if options.content is not None: + kwargs["data"] = options.content + if options.data is not None or options.files is not None: + kwargs["data"] = self._get_data_and_files_kwargs(options.data, options.files) + kwargs["cookies"] = self._get_cookies_kwarg( + {} if options.disregard_default_cookies else self._cookies, options.cookies + ) + kwargs["headers"] = self._get_headers_kwarg( + {} if options.disregard_default_headers else self._headers, options.headers + ) + if options.json is not shared_http._JsonEmptySentinel and ( + options.content is not None or options.data is not None or options.files is not None + ): + raise ValueError('Cannot specify "json" alongside "content", "data", or "files"') + if options.json is not shared_http._JsonEmptySentinel: + kwargs["json"] = options.json + if options.timeout: + kwargs["timeout"] = aiohttp.ClientTimeout(total=options.timeout, sock_connect=min(30, options.timeout)) + return kwargs + + def get(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs(options if options is not None else shared_http._EMPTY_OPTIONS) + return _WrappedHTTPResponse(self._http_session.get(url, **kwargs)) + + def post(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs(options if options is not None else shared_http._EMPTY_OPTIONS) + return _WrappedHTTPResponse(self._http_session.post(url, **kwargs)) + + def put(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs(options if options is not None else shared_http._EMPTY_OPTIONS) + return _WrappedHTTPResponse(self._http_session.put(url, **kwargs)) + + def delete(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs(options if options is not None else shared_http._EMPTY_OPTIONS) + return _WrappedHTTPResponse(self._http_session.delete(url, **kwargs)) + + def request(self, method: shared_http.HTTPMethod, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs(options if options is not None else shared_http._EMPTY_OPTIONS) + return _WrappedHTTPResponse(self._http_session.request(method.name, url, **kwargs)) diff --git a/scratchattach/async_api/primitives/utils.py b/scratchattach/async_api/primitives/utils.py new file mode 100644 index 00000000..ef72bf6d --- /dev/null +++ b/scratchattach/async_api/primitives/utils.py @@ -0,0 +1,112 @@ +from __future__ import annotations +from collections.abc import Callable +from typing import Union, ParamSpec, TypeVar, Generic, Any, cast, Optional, overload, Literal +import time + +CTYPES_PRESENT = False +import asyncio +from collections.abc import Awaitable, Coroutine + + +async def sleep_prim(delay: Union[int, float]): + await asyncio.sleep(delay) + + +P = ParamSpec("P") +O = TypeVar("O", covariant=True) + + +class Task(Generic[P, O]): + function: Callable[P, O] + args: Any + kwargs: Any + available: bool + + +class LaunchedTask(Generic[P, O]): + task: Task[P, O] + _task: asyncio.Task[Any] + + +def create_task(function: Callable[P, O], *args: P.args, **kwargs: P.kwargs) -> Task[P, O]: + task: Task[P, O] = Task() + task.function = function + task.args = args + task.kwargs = kwargs + task.available = True + return task + + +async def gather_concurrently_prim(*tasks: Task[Any, Awaitable[O]]) -> list[O]: + for task in tasks: + if not task.available: + raise ValueError("Task is already used.") + task.available = False + return await asyncio.gather(*(task.function(*task.args, **task.kwargs) for task in tasks)) + + +A = TypeVar("A") +B = TypeVar("B") + + +async def launch_concurrently_prim(task: Task[P, Coroutine[A, B, O]]) -> LaunchedTask[P, Coroutine[A, B, O]]: + _task = asyncio.create_task(task.function(*task.args, **task.kwargs)) + launched_task: LaunchedTask[P, Coroutine[A, B, O]] = LaunchedTask() + launched_task.task = task + launched_task._task = _task + return launched_task + + +@overload +async def join_launched_task_prim(task: LaunchedTask[P, Coroutine[Any, Any, O]]) -> O: + pass + + +@overload +async def join_launched_task_prim(task: LaunchedTask[P, Coroutine[Any, Any, O]], timeout: Union[float, int]) -> Optional[O]: + pass + + +async def join_launched_task_prim( + task: LaunchedTask[P, Coroutine[Any, Any, O]], timeout: Optional[Union[float, int]] = None +) -> Optional[O]: + try: + return await asyncio.wait_for(asyncio.shield(task._task), timeout) + except TimeoutError: + return None + + +@overload +async def kill_launched_task_prim(task: LaunchedTask[P, O], *, exception_interval: Union[float, int] = 0.1) -> Literal[True]: + """ + Sends exceptions to the underlying concurrency primitive. + May also try to use the recommended way of cancelling the primitive if there is one. + Returns whether the task was actually killed. + """ + + +@overload +async def kill_launched_task_prim( + task: LaunchedTask[P, O], timeout: Union[float, int], *, exception_interval: Union[float, int] = 0.1 +) -> bool: + """ + Sends exceptions to the underlying concurrency primitive. + May also try to use the recommended way of cancelling the primitive if there is one. + Returns whether the task was actually killed. + """ + + +async def kill_launched_task_prim( + task: LaunchedTask[P, O], timeout: Optional[Union[float, int]] = None, *, exception_interval: Union[float, int] = 0.1 +) -> bool: + has_timeout, timeout_end = (True, time.time() + timeout) if timeout is not None else (False, None) + if task._task.cancel(): + return True + while not has_timeout or (timeout_end is not None and time.time() <= timeout_end): + if not task._task.done(): + break + task._task.set_exception(SystemExit) + await asyncio.sleep(exception_interval) + if has_timeout and timeout_end is not None and (time.time() > timeout_end): + return False + return True diff --git a/scratchattach/async_api/site/_base.py b/scratchattach/async_api/site/_base.py new file mode 100644 index 00000000..972a3914 --- /dev/null +++ b/scratchattach/async_api/site/_base.py @@ -0,0 +1,196 @@ +from __future__ import annotations +from abc import ABC, abstractmethod +from typing import TypeVar, Optional, Self, Union, Any, Generic, TypeAlias, cast, overload, Literal +import json +from collections.abc import Callable, Awaitable +from scratchattach.utils import exceptions, commons, optional_async +from scratchattach._shared import http as shared_http +from . import session +from ..primitives import http + +D = TypeVar("D") +C = TypeVar("C", bound="BaseSiteComponent") + + +class BaseSiteComponent(ABC, Generic[D]): + _session: session.Session | session.UnauthSession + update_api: str + + async def update(self): + """ + Updates the attributes of the object by performing an API response. Returns True if the update was successful. + """ + async with self._session.http_session.request( + self.update_method, self.update_api, shared_http.options().timeout(10).value + ) as response: + if response.status_code == 429: + return "429" + response_json = await response.json() + if response_json == {"response": "Too many requests"}: + return "429" + assert isinstance(response_json, dict) + if "code" in response_json: + return False + return self._update_from_data(cast(D, response_json)) + + async def updated(self) -> Self: + await self.update() + return self + + @abstractmethod + def _update_from_data(self, data: D) -> bool: + """ + Parses the API response that is fetched in the update-method. Class specific, must be overridden in classes inheriting from this one. + """ + + def _assert_auth(self) -> session.Session: + if isinstance(self._session, session.UnauthSession): + raise exceptions.Unauthenticated( + "You need to use session.connect_xyz (NOT get_xyz) in order to perform this operation." + ) + return self._session + + @classmethod + def _get_object( + cls, identificator_name, identificator, NotFoundException, session: session.Session | session.UnauthSession + ) -> Self: + from scratchattach.site import project + + try: + use_class: type = cls + if cls is project.PartialProject: + use_class = project.Project + assert issubclass(use_class, cls) + _object = use_class(**{identificator_name: identificator, "_session": session}) + r = _object.update() + if r == "429": + raise exceptions.Response429( + "Your network is blocked or rate-limited by Scratch.\nIf you're using an online IDE like replit.com, try running the code on your computer." + ) + if not r: + if cls is project.PartialProject: + _object = project.PartialProject( + **{identificator_name: identificator, "shared": False, "_session": session} + ) + assert isinstance(_object, cls) + return _object + else: + raise NotFoundException + else: + return _object + except KeyError as e: + raise NotFoundException(f"Key error at key {e} when reading API response") + except Exception as e: + raise e + + def _make_linked_object(self, identificator_id, identificator, cls: type[C], not_found_exception) -> C: + """ + Internal function for making a linked object (authentication kept) based on an identificator (like a project id or username) + Class must inherit from BaseSiteComponent + """ + return cls._get_object(identificator_id, identificator, not_found_exception, self._session) + + @classmethod + def parse_object_list( + cls, raw: list[D], /, session: session.Session | session.UnauthSession, primary_key: str = "id" + ) -> list[Self]: + results = [] + for raw_dict in raw: + try: + _obj = cls(**{primary_key: cast(dict[str, Any], raw_dict)[primary_key], "_session": session}) + _obj._update_from_data(raw_dict) + results.append(_obj) + except Exception as e: + print("Warning raised by scratchattach: failed to parse ", raw_dict, "error", e) + return results + + def supply_data_dict(self, data: D) -> bool: + return self._update_from_data(data) + + update_method: shared_http.HTTPMethod = shared_http.HTTPMethod.GET + "\n HTTP method for getting updated information for this component\n " + + +F = TypeVar("F") + + +@overload +async def api_iterative_data( + fetch_func: Callable[[int, int], Awaitable[list[F] | None]], + limit: int, + offset: int, + max_req_limit: int = 40, + unpack: Literal[True] = True, +) -> list[F]: ... + + +@overload +async def api_iterative_data( + fetch_func: Callable[[int, int], Awaitable[F | None]], + limit: int, + offset: int, + max_req_limit: int = 40, + unpack: Literal[False] = False, +) -> list[F]: ... + + +async def api_iterative_data( + fetch_func: Callable[[int, int], Any], limit: int, offset: int, max_req_limit: int = 40, unpack: bool = True +) -> list: + """ + Iteratively gets data by calling fetch_func with a moving offset and a limit. + Once fetch_func returns None, the retrieval is completed. + """ + if limit is None: + limit = max_req_limit + end = offset + limit + api_data: list[Any] = [] + for offs in range(offset, end, max_req_limit): + data = await fetch_func(offs, max_req_limit) + if data is None: + break + if unpack: + api_data.extend(data) + else: + api_data.append(data) + if len(data) < max_req_limit: + break + return api_data[:limit] + + +async def api_iterative( + session: session.Session | session.UnauthSession, + url: str, + *, + limit: int, + offset: int, + max_req_limit: int = 40, + add_params: str = "", + _headers: Optional[dict] = None, + cookies: Optional[dict] = None, +) -> list[F]: + """ + Function for getting data from one of Scratch's iterative JSON API endpoints (like /users//followers, or /users//projects) + """ + if _headers is None: + _headers = commons.headers.copy() + if cookies is None: + cookies = {} + if offset < 0: + raise exceptions.BadRequest("offset parameter must be >= 0") + if limit < 0: + raise exceptions.BadRequest("limit parameter must be >= 0") + + async def fetch(off: int, lim: int) -> list[F] | None: + """ + Performs a single API request + """ + async with session.http_session.get(f"{url}?limit={lim}&offset={off}{add_params}") as response: + resp = cast(list[F], await response.json()) + if not resp: + return None + if resp == {"code": "BadRequest", "message": ""}: + raise exceptions.BadRequest("The arguments passed are invalid") + return resp + + return await api_iterative_data(fetch, limit, offset, max_req_limit=max_req_limit, unpack=True) diff --git a/scratchattach/async_api/site/activity.py b/scratchattach/async_api/site/activity.py new file mode 100644 index 00000000..d762f62a --- /dev/null +++ b/scratchattach/async_api/site/activity.py @@ -0,0 +1,372 @@ +"""Activity and CloudActivity class""" + +from __future__ import annotations +import html +import warnings +from dataclasses import dataclass +from typing import Optional, Any +from enum import Enum +from bs4 import Tag +from . import user, project, studio, session, forum, comment +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions + + +class ActivityTypes(Enum): + loveproject = "loveproject" + favoriteproject = "favoriteproject" + becomecurator = "becomecurator" + followuser = "followuser" + followstudio = "followstudio" + shareproject = "shareproject" + remixproject = "remixproject" + becomeownerstudio = "becomeownerstudio" + addcomment = "addcomment" + curatorinvite = "curatorinvite" + userjoin = "userjoin" + studioactivity = "studioactivity" + forumpost = "forumpost" + updatestudio = "updatestudio" + createstudio = "createstudio" + promotetomanager = "promotetomanager" + updateprofile = "updateprofile" + removeprojectfromstudio = "removeprojectfromstudio" + addprojecttostudio = "addprojecttostudio" + performaction = "performaction" + + +@dataclass +class Activity(BaseSiteComponent): + """ + Represents a Scratch activity (message or other user page activity) + """ + + _session: Optional[session.Session] = None + raw: Any = None + id: Optional[int] = None + actor_username: Optional[str] = None + project_id: Optional[int] = None + gallery_id: Optional[int] = None + username: Optional[str] = None + followed_username: Optional[str] = None + recipient_username: Optional[str] = None + title: Optional[str] = None + project_title: Optional[str] = None + gallery_title: Optional[str] = None + topic_title: Optional[str] = None + topic_id: Optional[int] = None + target_name: Optional[str] = None + target_id: Optional[int | str] = None + parent_title: Optional[str] = None + parent_id: Optional[int] = None + comment_type: Optional[int] = None + comment_obj_id: Optional[int] = None + comment_obj_title: Optional[str] = None + comment_id: Optional[int] = None + comment_fragment: Optional[str] = None + changed_fields: Optional[dict[str, str]] = None + is_reshare: Optional[bool] = None + datetime_created: Optional[str] = None + time: Any = None + type: Optional[ActivityTypes] = None + + def __repr__(self): + return f"Activity({repr(self.raw)})" + + def __str__(self): + return "-A " + " ".join(self.parts) + + def _parts_simple(self, verb: str, obj: str): + return [str(self.actor_username), verb, obj] + + def _parts_comment(self) -> list[str]: + ret = [str(self.actor_username), "commented on"] + if self.comment_type not in (0, 1, 2): + raise ValueError(f"Unknown comment type: {self.comment_type}") + ret.append( + { + 0: f"-P {self.comment_obj_title!r} ({self.comment_obj_id}", + 1: f"-U {self.comment_obj_title}", + 2: f"-S {self.comment_obj_title!r} ({self.comment_obj_id}", + }[self.comment_type] + ) + ret[-1] += f"#{self.comment_id})" + ret.append(str(html.unescape(str(self.comment_fragment)))) + return ret + + @property + def parts(self): + """ + Return format: [actor username] + N * [action, object] + :return: A list of parts of the message. Join the parts to get a readable version, which is done with str(activity) + """ + SIMPLE_SOLNS = { + ActivityTypes.loveproject: ("loved", f"-P {self.title!r} ({self.project_id})"), + ActivityTypes.favoriteproject: ("favorited", f"-P {self.project_title!r} ({self.project_id})"), + ActivityTypes.becomecurator: ("now curating", f"-S {self.title!r} ({self.gallery_id})"), + ActivityTypes.followuser: ("followed", f"-U {self.followed_username}"), + ActivityTypes.followstudio: ("followed", f"-S {self.title!r} ({self.gallery_id})"), + ActivityTypes.shareproject: ( + "reshared" if self.is_reshare else "shared", + f"-P {self.title!r} ({self.project_id})", + ), + ActivityTypes.remixproject: ( + "remixed", + f"-P {self.parent_title!r} ({self.parent_id}) as -P {self.title!r} ({self.project_id})", + ), + ActivityTypes.becomeownerstudio: ("became owner of", f"-S {self.gallery_title!r} ({self.gallery_id})"), + ActivityTypes.curatorinvite: ("invited you to curate", f"-S {self.title!r} ({self.gallery_id})"), + ActivityTypes.forumpost: ("posted in", f"-F {self.topic_title} ({self.topic_id})"), + ActivityTypes.updatestudio: ("updated", f"-S {self.gallery_title} ({self.gallery_id})"), + ActivityTypes.createstudio: ("created", f"-S {self.gallery_title} ({self.gallery_id})"), + None: (), + } + if args := SIMPLE_SOLNS.get(self.type): + return self._parts_simple(*args) + match self.type: + case ActivityTypes.addcomment: + return self._parts_comment() + case ActivityTypes.userjoin: + return [str(self.actor_username), "joined Scratch"] + case ActivityTypes.studioactivity: + return [str(self.actor_username), "Studio activity", "", f"-S {self.title!r} ({self.gallery_id})"] + case ActivityTypes.promotetomanager: + return [ + str(self.actor_username), + "promoted", + f"-U {self.recipient_username}", + "in", + f"-S {self.gallery_title} ({self.gallery_id})", + ] + case ActivityTypes.updateprofile: + return [str(self.actor_username), "updated their profile.", f"Changed fields: {self.changed_fields}"] + case ActivityTypes.removeprojectfromstudio: + return [ + f"{self.actor_username}", + "removed", + f"-P {self.project_title} ({self.project_id})", + "from", + f"-S {self.gallery_title} ({self.gallery_id})", + ] + case ActivityTypes.addprojecttostudio: + return [ + f"{self.actor_username}", + "added", + f"-P {self.project_title} ({self.project_id})", + "to", + f"-S {self.gallery_title} ({self.gallery_id})", + ] + case ActivityTypes.performaction: + return [f"{self.actor_username}", "performed an action"] + case _: + raise NotImplementedError( + f"Activity type {self.type!r} is not implemented!\nself.raw={self.raw!r}\nRaise an issue on github: https://github.com/TimMcCool/scratchattach/issues" + ) + + def update(self): + print("Warning: Activity objects can't be updated") + return False + + def _update_from_data(self, data): + self.raw = data + self._session = data.get("_session", self._session) + self.raw = data.get("raw", self.raw) + self.id = data.get("id", self.id) + self.actor_username = data.get("actor_username", self.actor_username) + self.project_id = data.get("project_id", self.project_id) + self.gallery_id = data.get("gallery_id", self.gallery_id) + self.username = data.get("username", self.username) + self.followed_username = data.get("followed_username", self.followed_username) + self.recipient_username = data.get("recipient_username", self.recipient_username) + self.title = data.get("title", self.title) + self.project_title = data.get("project_title", self.project_title) + self.gallery_title = data.get("gallery_title", self.gallery_title) + self.topic_title = data.get("topic_title", self.topic_title) + self.topic_id = data.get("topic_id", self.topic_id) + self.target_name = data.get("target_name", self.target_name) + self.target_id = data.get("target_id", self.target_id) + self.parent_title = data.get("parent_title", self.parent_title) + self.parent_id = data.get("parent_id", self.parent_id) + self.comment_type = data.get("comment_type", self.comment_type) + self.comment_obj_id = data.get("comment_obj_id", self.comment_obj_id) + self.comment_obj_title = data.get("comment_obj_title", self.comment_obj_title) + self.comment_id = data.get("comment_id", self.comment_id) + self.comment_fragment = data.get("comment_fragment", self.comment_fragment) + self.changed_fields = data.get("changed_fields", self.changed_fields) + self.is_reshare = data.get("is_reshare", self.is_reshare) + self.datetime_created = data.get("datetime_created", self.datetime_created) + self.time = data.get("time", self.time) + _type = data.get("type", self.type) + if _type == "becomehoststudio": + self.type = ActivityTypes.becomeownerstudio + elif _type: + self.type = ActivityTypes[_type] + return True + + def _update_from_json(self, data: dict): + """ + Update using JSON, used in the classroom API. + """ + activity_type = data["type"] + _time = data.get("datetime_created") + if "actor" in data: + self.username = data["actor"]["username"] + else: + self.username = data.get("actor_username") + self.recipient_username = None + if recipient := data.get("recipient"): + self.recipient_username = recipient["username"] + elif ru := data.get("recipient_username"): + self.recipient_username = ru + elif project_creator := data.get("project_creator"): + self.recipient_username = project_creator["username"] + self.actor_username = self.username + self.raw = data + self.datetime_created = _time + self.type = { + 0: ActivityTypes.followuser, + 1: ActivityTypes.followstudio, + 2: ActivityTypes.loveproject, + 3: ActivityTypes.favoriteproject, + 7: ActivityTypes.addprojecttostudio, + 8: ActivityTypes.shareproject, + 9: ActivityTypes.shareproject, + 10: ActivityTypes.shareproject, + 11: ActivityTypes.remixproject, + 13: ActivityTypes.createstudio, + 15: ActivityTypes.updatestudio, + 16: ActivityTypes.removeprojectfromstudio, + 17: ActivityTypes.removeprojectfromstudio, + 18: ActivityTypes.removeprojectfromstudio, + 19: ActivityTypes.removeprojectfromstudio, + 20: ActivityTypes.promotetomanager, + 21: ActivityTypes.promotetomanager, + 22: ActivityTypes.promotetomanager, + 23: ActivityTypes.updateprofile, + 24: ActivityTypes.updateprofile, + 25: ActivityTypes.updateprofile, + 26: ActivityTypes.addcomment, + 27: ActivityTypes.addcomment, + None: ActivityTypes.performaction, + }.get(activity_type, ActivityTypes.performaction) + self.followed_username = data.get("followed_username", self.followed_username) + self.gallery_id = data.get("gallery", self.gallery_id) + self.project_id = data.get("project", self.project_id) + self.is_reshare = data.get("is_reshare", self.is_reshare) + self.comment_fragment = data.get("comment_fragment", self.comment_fragment) + self.comment_type = data.get("comment_type", self.comment_type) + self.comment_obj_id = data.get("comment_obj_id", self.comment_obj_id) + self.comment_obj_title = data.get("comment_obj_title", self.comment_obj_title) + self.comment_id = data.get("comment_id", self.comment_id) + self.parent_id = data.get("parent", self.parent_id) + if self.parent_id: + warnings.warn( + f"This may be incorrectly implemented.\nRaw data: {data}\nPlease raise an issue on gh: https://github.com/TimMcCool/scratchattach/issues" + ) + if self.type == ActivityTypes.updateprofile: + self.changed_fields = data.get("changed_fields", {}) + + def _update_from_html(self, data: Tag): + self.raw = data + _time = data.find("div").find("span").find_next().find_next().text.strip() + if "\xa0" in _time: + while "\xa0" in _time: + _time = _time.replace("\xa0", " ") + self.datetime_created = _time + self.actor_username = data.find("div").find("span").text + self.target_name = data.find("div").find("span").find_next().text + self.target_link = data.find("div").find("span").find_next()["href"] + self.target_id = data.find("div").find("span").find_next()["href"].split("/")[-2] + _type = data.find("div").find_all("span")[0].next_sibling.strip() + if _type == "loved": + self.type = ActivityTypes.loveproject + elif _type == "favorited": + self.type = ActivityTypes.favoriteproject + elif "curator" in _type: + self.type = ActivityTypes.becomecurator + elif "shared" in _type: + self.type = ActivityTypes.shareproject + elif "is now following" in _type: + if "users" in self.target_link: + self.type = ActivityTypes.followuser + else: + self.type = ActivityTypes.followstudio + return True + + def actor(self): + """ + Returns the user that performed the activity as User object + """ + return self._make_linked_object("username", self.actor_username, user.User, exceptions.UserNotFound) + + def target_project(self) -> Optional[project.Project]: + if self.target_id: + return self._make_linked_object("id", self.target_id, project.Project, exceptions.ProjectNotFound) + if self.project_id: + return self._make_linked_object("id", self.project_id, project.Project, exceptions.ProjectNotFound) + return None + + def target_studio(self) -> Optional[studio.Studio]: + if self.target_id: + return self._make_linked_object("id", self.target_id, studio.Studio, exceptions.StudioNotFound) + if self.gallery_id: + return self._make_linked_object("id", self.gallery_id, studio.Studio, exceptions.StudioNotFound) + return None + + def target_user(self) -> Optional[user.User]: + if self.username: + return self._make_linked_object("username", self.username, user.User, exceptions.UserNotFound) + if self.target_name: + return self._make_linked_object("username", self.target_name, user.User, exceptions.UserNotFound) + if self.followed_username: + return self._make_linked_object("username", self.followed_username, user.User, exceptions.UserNotFound) + if self.recipient_username: + return self._make_linked_object("username", self.recipient_username, user.User, exceptions.UserNotFound) + return None + + def target_comment(self) -> Optional[comment.Comment]: + if self.comment_type == 0: + if self.comment_obj_id is None: + return None + if self._session is not None: + _proj = self._session.connect_project(self.comment_obj_id) + else: + _proj = project.Project(id=self.comment_obj_id) + return _proj.comment_by_id(self.comment_id) + elif self.comment_type == 1: + return user.User(username=self.comment_obj_title, _session=self._session).comment_by_id(self.comment_id) + elif self.comment_type == 2: + return user.User(id=self.comment_obj_id, _session=self._session).comment_by_id(self.comment_id) + else: + return None + + def target(self): + """ + Returns the activity's target (depending on the activity, this is either a User, Project, Studio or Comment object). + May also return None if the activity type is unknown. + """ + if self.type is None: + return None + _type = self.type.value + if self.type in ( + ActivityTypes.addprojecttostudio, + ActivityTypes.favoriteproject, + ActivityTypes.loveproject, + ActivityTypes.remixproject, + ActivityTypes.removeprojectfromstudio, + ActivityTypes.shareproject, + ): + return self.target_project() + if self.type in (ActivityTypes.becomecurator, ActivityTypes.followstudio): + if ret := self.target_studio(): + return ret + return self.target_user() + if self.type in (ActivityTypes.followuser, ActivityTypes.curatorinvite) or self.recipient_username: + return self.target_user() + if self.type == ActivityTypes.addcomment: + if ret := self.target_comment(): + return ret + raise ValueError(f"Either {self.comment_type} is an invalid comment type, or the linked target could not be found") + if _type == "forumpost": + return forum.ForumTopic(id=603418, _session=self._session, title=self.title) + return None diff --git a/scratchattach/async_api/site/alert.py b/scratchattach/async_api/site/alert.py new file mode 100644 index 00000000..1ae9e2fc --- /dev/null +++ b/scratchattach/async_api/site/alert.py @@ -0,0 +1,177 @@ +from __future__ import annotations +import json +import pprint +import warnings +from dataclasses import dataclass, field, KW_ONLY +from datetime import datetime +from typing_extensions import TYPE_CHECKING, Any, Optional, Union, Self +from . import user, project, studio, comment, session +from scratchattach.utils import enums + +if TYPE_CHECKING: + ... + + +@dataclass +class EducatorAlert: + """ + Represents an alert for student activity, viewable at https://scratch.mit.edu/site-api/classrooms/alerts/ + + Attributes: + model: The type of alert (presumably); should always equal "educators.educatoralert" in this class + type: An integer that identifies the type of alert, differentiating e.g. against bans or autoban or censored comments etc + raw: The raw JSON data from the API + id: The ID of the alert (internally called 'pk' by scratch, not sure what this is for) + time_read: The time the alert was read + time_created: The time the alert was created + target: The user that the alert is about (the student) + actor: The user that created the alert (the admin) + target_object: The object that the alert is about (e.g. a project, studio, or comment) + notification_type: not sure what this is for, but inferred from the scratch HTML reference + """ + + _: KW_ONLY + target: user.User + actor: user.User + target_object: Optional[Union[project.Project, studio.Studio, comment.Comment, studio.Studio]] + notification_type: str + _session: Optional[session.Session] + model: str = "educators.educatoralert" + type: int = -1 + raw: dict = field(repr=False, default_factory=dict) + id: int = -1 + time_read: datetime = datetime.fromtimestamp(0.0) + time_created: datetime = datetime.fromtimestamp(0.0) + + @classmethod + def from_json(cls, data: dict[str, Any], _session: Optional[session.Session] = None) -> Self: + """ + Load an EducatorAlert from a JSON object. + + Arguments: + data (dict): The JSON object + _session (session.Session): The session object used to load this data, to 'connect' to the alerts rather than just 'get' them + + Returns: + EducatorAlert: The loaded EducatorAlert object + """ + model = data.get("model") + assert isinstance(model, str) + alert_id = data.get("pk") + assert isinstance(alert_id, int) + fields = data.get("fields") + assert isinstance(fields, dict) + time_read_raw = fields.get("educator_datetime_read") + assert isinstance(time_read_raw, str) + time_read: datetime = datetime.fromisoformat(time_read_raw) + admin_action = fields.get("admin_action") + assert isinstance(admin_action, dict) + time_created_raw = admin_action.get("datetime_created") + assert isinstance(time_created_raw, str) + time_created: datetime = datetime.fromisoformat(time_created_raw) + alert_type = admin_action.get("type") + assert isinstance(alert_type, int) + target_data = admin_action.get("target_user") + assert isinstance(target_data, dict) + target = user.User( + username=target_data.get("username"), + id=target_data.get("pk"), + icon_url=target_data.get("thumbnail_url"), + admin=target_data.get("admin", False), + _session=_session, + ) + actor_data = admin_action.get("actor") + assert isinstance(actor_data, dict) + actor = user.User( + username=actor_data.get("username"), + id=actor_data.get("pk"), + icon_url=actor_data.get("thumbnail_url"), + admin=actor_data.get("admin", False), + _session=_session, + ) + object_id = admin_action.get("object_id") + assert isinstance(object_id, int) + target_object: project.Project | studio.Studio | comment.Comment | None = None + extra_data: dict[str, Any] = json.loads(admin_action.get("extra_data", "{}")) + notification_type: str = "" + if "project_title" in extra_data: + target_object = project.Project(id=object_id, title=extra_data["project_title"], _session=_session) + elif "comment_content" in extra_data: + comment_data: dict[str, Any] = extra_data["comment_content"] + content: str | None = comment_data.get("content") + comment_obj_id: int | None = comment_data.get("comment_obj_id") + comment_type: int | None = comment_data.get("comment_type") + if comment_type == 0: + comment_source_type = comment.CommentSource.PROJECT + elif comment_type == 1: + comment_source_type = comment.CommentSource.USER_PROFILE + else: + comment_source_type = comment.CommentSource.STUDIO + warnings.warn( + f'The parser was not able to recognise the "comment_type" of {comment_type} in the alert JSON response.\nFull response: \n{pprint.pformat(data)}.\n\nPlease draft an issue on github: https://github.com/TimMcCool/scratchattach/issues, providing this whole error message. This will allow us to implement an incomplete part of this parser' + ) + target_object = comment.Comment( + id=object_id, content=content, source=comment_source_type, source_id=comment_obj_id, _session=_session + ) + elif "gallery_title" in extra_data: + target_object = studio.Studio(id=object_id, title=extra_data["gallery_title"], _session=_session) + elif "notification_type" in extra_data: + notification_type = extra_data["notification_type"] + else: + warnings.warn( + f'The parser was not able to recognise the "extra_data" in the alert JSON response.\nFull response: \n{pprint.pformat(data)}.\n\nPlease draft an issue on github: https://github.com/TimMcCool/scratchattach/issues, providing this whole error message. This will allow us to implement an incomplete part of this parser' + ) + return cls( + id=alert_id, + model=model, + type=alert_type, + raw=data, + time_read=time_read, + time_created=time_created, + target=target, + actor=actor, + target_object=target_object, + notification_type=notification_type, + _session=_session, + ) + + def __str__(self): + return f"EducatorAlert: {self.message}" + + @property + def alert_type(self) -> enums.AlertType: + """ + Get an associated AlertType object for this alert (based on the type index) + """ + alert_type = enums.AlertTypes.find(self.type) + if not alert_type: + alert_type = enums.AlertTypes.default.value + return alert_type + + @property + def message(self): + """ + Format the alert message using the alert type's message template, as it would be on the website. + """ + raw_message = self.alert_type.message + comment_content = "" + if isinstance(self.target_object, comment.Comment): + comment_content = self.target_object.content + return raw_message.format( + username=self.target.username, + project=self.target_object_title, + studio=self.target_object_title, + notification_type=self.notification_type, + comment=comment_content, + ) + + @property + def target_object_title(self): + """ + Get the title of the target object (if applicable) + """ + if isinstance(self.target_object, project.Project): + return self.target_object.title + if isinstance(self.target_object, studio.Studio): + return self.target_object.title + return None diff --git a/scratchattach/async_api/site/backpack_asset.py b/scratchattach/async_api/site/backpack_asset.py new file mode 100644 index 00000000..69da6c8e --- /dev/null +++ b/scratchattach/async_api/site/backpack_asset.py @@ -0,0 +1,104 @@ +from __future__ import annotations +import json +import time +import logging +import warnings +from dataclasses import dataclass +from typing import Any, TYPE_CHECKING +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions +from scratchattach.utils.requests import requests + +if TYPE_CHECKING: + from scratchattach import session + + +@dataclass +class BackpackAsset(BaseSiteComponent): + """ + Represents an asset from the backpack. + + Attributes: + + :.id: + + :.type: The asset type (costume, script etc.) + + :.mime: The format in which the content of the backpack asset is saved + + :.name: The name of the backpack asset + + :.filename: Filename of the file containing the content of the backpack asset + + :.thumbnail_url: Link that leads to the asset's thumbnail (the image shown in the backpack UI) + + :.download_url: Link that leads to a file containing the content of the backpack asset + """ + + id: str + _session: session.Session | None = None + type: str | None = None + mime: str | None = None + name: str | None = None + filename: str | None = None + thumbnail_url: str | None = None + download_url: str | None = None + + def __repr__(self) -> str: + return f"BackpackAsset({self.filename})" + + def update(self): + warnings.warn("Warning: BackpackAsset objects can't be updated") + return False + + def _update_from_data(self, data: dict[str, str]) -> bool: + self.id = data.get("id", self.id) + self.type = data.get("type", self.type) + self.mime = data.get("mime", self.mime) + self.name = data.get("name", self.name) + self.filename = data.get("body", self.filename) + if "thumbnail" in data: + self.thumbnail_url = "https://backpack.scratch.mit.edu/" + data["thumbnail"] + if "body" in data: + self.download_url = "https://backpack.scratch.mit.edu/" + data["body"] + return True + + @property + def _data_bytes(self) -> bytes: + try: + with requests.no_error_handling(): + return requests.get(self.download_url).content + except Exception as e: + raise exceptions.FetchError(f"Failed to download asset: {e}") + + @property + def file_ext(self): + return self.filename.split(".")[-1] + + @property + def is_json(self): + return self.file_ext == "json" + + @property + def data(self) -> dict | list | int | None | str | bytes | float: + if self.is_json: + return json.loads(self._data_bytes) + else: + return self._data_bytes + + def download(self, *, fp: str = ""): + """ + Downloads the asset content to the given directory. The given filename is equal to the value saved in the .filename attribute. + + Args: + fp (str): The path of the directory the file will be saved in. + """ + if not (fp.endswith("/") or fp.endswith("\\")): + fp = fp + "/" + open(f"{fp}{self.filename}", "wb").write(self._data_bytes) + + def delete(self): + self._assert_auth() + return requests.delete( + f"https://backpack.scratch.mit.edu/{self._session.username}/{self.id}", headers=self._session._headers, timeout=10 + ).json() diff --git a/scratchattach/async_api/site/browser_cookie3_stub.py b/scratchattach/async_api/site/browser_cookie3_stub.py new file mode 100644 index 00000000..d535effe --- /dev/null +++ b/scratchattach/async_api/site/browser_cookie3_stub.py @@ -0,0 +1,46 @@ +import http.cookiejar +from typing import Optional + + +def chrome(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def chromium(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def firefox(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def opera(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def edge(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def brave(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def vivaldi(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def safari(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def lynx(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def w3m(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def load() -> http.cookiejar.CookieJar: + return NotImplemented diff --git a/scratchattach/async_api/site/browser_cookies.py b/scratchattach/async_api/site/browser_cookies.py new file mode 100644 index 00000000..7f50ff84 --- /dev/null +++ b/scratchattach/async_api/site/browser_cookies.py @@ -0,0 +1,64 @@ +from typing import Optional, TYPE_CHECKING +from typing_extensions import assert_never +from http.cookiejar import CookieJar +from enum import Enum, auto + +browsercookie_err = None +try: + if TYPE_CHECKING: + from . import browser_cookie3_stub as browser_cookie3 + else: + import browser_cookie3 +except Exception as e: + browsercookie = None + browsercookie_err = e + + +class Browser(Enum): + ANY = auto() + FIREFOX = auto() + CHROME = auto() + EDGE = auto() + SAFARI = auto() + CHROMIUM = auto() + VIVALDI = auto() + EDGE_DEV = auto() + + +FIREFOX = Browser.FIREFOX +CHROME = Browser.CHROME +EDGE = Browser.EDGE +SAFARI = Browser.SAFARI +CHROMIUM = Browser.CHROMIUM +VIVALDI = Browser.VIVALDI +ANY = Browser.ANY +EDGE_DEV = Browser.EDGE_DEV + + +def cookies_from_browser(browser: Browser = ANY) -> dict[str, str]: + """ + Import cookies from browser to login + """ + if not browser_cookie3: + raise browsercookie_err or ModuleNotFoundError() + cookies: Optional[CookieJar] = None + if browser is Browser.ANY: + cookies = browser_cookie3.load() + elif browser is Browser.FIREFOX: + cookies = browser_cookie3.firefox() + elif browser is Browser.CHROME: + cookies = browser_cookie3.chrome() + elif browser is Browser.EDGE: + cookies = browser_cookie3.edge() + elif browser is Browser.SAFARI: + cookies = browser_cookie3.safari() + elif browser is Browser.CHROMIUM: + cookies = browser_cookie3.chromium() + elif browser is Browser.VIVALDI: + cookies = browser_cookie3.vivaldi() + elif browser is Browser.EDGE_DEV: + raise ValueError("EDGE_DEV is not supported anymore.") + else: + assert_never(browser) + assert isinstance(cookies, CookieJar) + return {cookie.name: cookie.value for cookie in cookies if "scratch.mit.edu" in cookie.domain and cookie.value} diff --git a/scratchattach/async_api/site/classroom.py b/scratchattach/async_api/site/classroom.py new file mode 100644 index 00000000..4cda253c --- /dev/null +++ b/scratchattach/async_api/site/classroom.py @@ -0,0 +1,417 @@ +from __future__ import annotations +import json +import warnings +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional, TYPE_CHECKING, Any, Callable +import bs4 +from bs4 import BeautifulSoup + +if TYPE_CHECKING: + from scratchattach.site.session import Session +from scratchattach.utils.commons import requests +from . import user, activity, typed_dicts +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions, commons +from scratchattach.utils.commons import headers + + +@dataclass +class Classroom(BaseSiteComponent): + title: str = "" + id: int = 0 + classtoken: str = "" + author: Optional[user.User] = None + about_class: str = "" + working_on: str = "" + is_closed: bool = False + datetime: datetime = datetime.fromtimestamp(0.0) + update_function: Callable = field(repr=False, default=requests.get) + _session: Optional[Session] = field(repr=False, default=None) + + def __post_init__(self): + if self.id: + self.update_api = f"https://api.scratch.mit.edu/classrooms/{self.id}" + elif self.classtoken: + self.update_api = f"https://api.scratch.mit.edu/classtoken/{self.classtoken}" + else: + raise KeyError(f"No class id or token provided! self.__dict__ = {self.__dict__!r}") + if self._session is None: + self._headers = commons.headers + self._cookies = {} + else: + self._headers = self._session._headers + self._cookies = self._session._cookies + self._json_headers = {**self._headers, "accept": "application/json", "Content-Type": "application/json"} + + def __str__(self) -> str: + return f"" + + def update(self): + try: + success = super().update() + except exceptions.ClassroomNotFound: + success = False + if not success: + response = requests.get(f"https://scratch.mit.edu/classes/{self.id}/") + soup = BeautifulSoup(response.text, "html.parser") + headings = soup.find_all("h1") + for heading in headings: + if heading.text == "Whoops! Our server is Scratch'ing its head": + raise exceptions.ClassroomNotFound(f"Classroom id {self.id} is not closed and cannot be found.") + title = soup.find("title").contents[0][: -len(" on Scratch")] + overviews = soup.find_all("p", {"class": "overview"}) + description, status = (overviews[0].text, overviews[1].text) + educator_username = None + pfx = "Scratch.INIT_DATA.PROFILE = {\n model: {\n id: '" + sfx = "',\n userId: " + for script in soup.find_all("script"): + if pfx in script.text: + educator_username = commons.webscrape_count(script.text, pfx, sfx, str) + ret: typed_dicts.ClassroomDict = { + "id": self.id, + "title": title, + "description": description, + "educator": {}, + "status": status, + "is_closed": True, + } + if educator_username: + ret["educator"]["username"] = educator_username + return self._update_from_data(ret) + return success + + def _update_from_data(self, data: typed_dicts.ClassroomDict): + self.id = int(data["id"]) + self.title = data["title"] + self.about_class = data["description"] + self.working_on = data["status"] + self.datetime = datetime.fromisoformat(data["date_start"]) + self.author = user.User(username=data["educator"]["username"], _session=self._session) + self.author.supply_data_dict(data["educator"]) + self.is_closed = bool(data["date_end"]) + return True + + def student_count(self) -> int: + text = requests.get(f"https://scratch.mit.edu/classes/{self.id}/", headers=self._headers).text + return commons.webscrape_count(text, "Students (", ")") + + def student_names(self, *, page=1) -> list[str]: + """ + Returns the student on the class. + + Keyword Arguments: + page: The page of the students that should be returned. + + Returns: + list: The usernames of the class students + """ + if self.is_closed: + ret = [] + response = requests.get(f"https://scratch.mit.edu/classes/{self.id}/") + soup = BeautifulSoup(response.text, "html.parser") + found = set("") + for result in soup.css.select("ul.scroll-content .user a"): + result_text = result.text.strip() + if result_text in found: + continue + found.add(result_text) + ret.append(result_text) + return ret + text = requests.get(f"https://scratch.mit.edu/classes/{self.id}/students/?page={page}", headers=self._headers).text + textlist = [i.split('/">')[0] for i in text.split(' list[int]: + """ + Returns the class studio on the class. + + Keyword Arguments: + page: The page of the students that should be returned. + + Returns: + list: The id of the class studios + """ + if self.is_closed: + ret = [] + response = requests.get(f"https://scratch.mit.edu/classes/{self.id}/") + soup = BeautifulSoup(response.text, "html.parser") + for result in soup.css.select("ul.scroll-content .gallery a[href]:not([class])"): + value = result["href"] + if not isinstance(value, str): + value = value[0] + ret.append(commons.webscrape_count(value, "/studios/", "/")) + return ret + text = requests.get(f"https://scratch.mit.edu/classes/{self.id}/studios/?page={page}", headers=self._headers).text + textlist = [int(i.split('/">')[0]) for i in text.split('\n None: + self._check_session() + requests.post( + f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, + cookies=self._cookies, + files={"file": thumbnail}, + ) + + def set_description(self, desc: str) -> None: + self._check_session() + response = requests.put( + f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, + cookies=self._cookies, + json={"description": desc}, + ) + try: + data = response.json() + if data["description"] == desc: + return + else: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def set_working_on(self, status: str) -> None: + self._check_session() + response = requests.put( + f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, + cookies=self._cookies, + json={"status": status}, + ) + try: + data = response.json() + if data["status"] == status: + return + else: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def set_title(self, title: str) -> None: + self._check_session() + response = requests.put( + f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, + cookies=self._cookies, + json={"title": title}, + ) + try: + data = response.json() + if data["title"] == title: + return + else: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def add_studio(self, name: str, description: str = "") -> None: + self._check_session() + requests.post( + "https://scratch.mit.edu/classes/create_classroom_gallery/", + json={"classroom_id": str(self.id), "classroom_token": self.classtoken, "title": name, "description": description}, + headers=self._headers, + cookies=self._cookies, + ) + + def reopen(self) -> None: + self._check_session() + response = requests.put( + f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, + cookies=self._cookies, + json={"visibility": "visible"}, + ) + try: + response.json() + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def close(self) -> None: + self._check_session() + response = requests.post( + f"https://scratch.mit.edu/site-api/classrooms/close_classroom/{self.id}/", + headers=self._headers, + cookies=self._cookies, + ) + try: + response.json() + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def register_student( + self, + username: str, + password: str = "", + birth_month: Optional[int] = None, + birth_year: Optional[int] = None, + gender: Optional[str] = None, + country: Optional[str] = None, + is_robot: bool = False, + ) -> None: + return register_by_token( + self.id, + self.classtoken, + username, + password, + birth_month or 1, + birth_year or 2000, + gender or "(Prefer not to say)", + country or "United+States", + is_robot, + ) + + def generate_signup_link(self): + if self.classtoken is not None: + return f"https://scratch.mit.edu/signup/{self.classtoken}" + self._check_session() + response = requests.get( + f"https://scratch.mit.edu/site-api/classrooms/generate_registration_link/{self.id}/", + headers=self._headers, + cookies=self._cookies, + ) + data = response.json() + if "reg_link" in data: + return data["reg_link"] + else: + raise exceptions.Unauthorized(f"{self._session} is not authorised to generate a signup link of {self}") + + def public_activity(self, *, limit=20): + """ + Returns: + list: The user's activity data as parsed list of scratchattach.activity.Activity objects + """ + if limit > 20: + warnings.warn("The limit is set to more than 20. There may be an error") + soup = BeautifulSoup( + requests.get(f"https://scratch.mit.edu/site-api/classrooms/activity/public/{self.id}/?limit={limit}").text, + "html.parser", + ) + activities = [] + source = soup.find_all("li") + for data in source: + _activity = activity.Activity(_session=self._session, raw=data) + _activity._update_from_html(data) + activities.append(_activity) + return activities + + def activity( + self, student: str = "all", mode: str = "Last created", page: Optional[int] = None + ) -> list[activity.Activity]: + """ + Get a list of private activity, only available to the class owner. + Returns: + list The private activity of users in the class + """ + self._check_session() + ascsort, descsort = commons.get_class_sort_mode(mode) + with requests.no_error_handling(): + try: + data = requests.get( + f"https://scratch.mit.edu/site-api/classrooms/activity/{self.id}/{student}/", + params={"page": page, "ascsort": ascsort, "descsort": descsort}, + headers=self._headers, + cookies=self._cookies, + ).json() + except json.JSONDecodeError: + return [] + _activity: list[activity.Activity] = [] + for activity_json in data: + _activity.append(activity.Activity(_session=self._session)) + _activity[-1]._update_from_json(activity_json) + return _activity + + +def get_classroom(class_id: str) -> Classroom: + """ + Gets a class without logging in. + + Args: + class_id (str): class id of the requested class + + Returns: + scratchattach.classroom.Classroom: An object representing the requested classroom + + Warning: + Any methods that require authentication will not work on the returned object. + + If you want to use these, get the user with :meth:`scratchattach.session.Session.connect_classroom` instead. + """ + warnings.warn( + "For methods that require authentication, use session.connect_classroom instead of get_classroom\nIf you want to remove this warning, use warnings.filterwarnings('ignore', category=scratchattach.ClassroomAuthenticationWarning)\nTo ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use `warnings.filterwarnings('ignore', category=scratchattach.GetAuthenticationWarning)`.", + exceptions.ClassroomAuthenticationWarning, + ) + return commons._get_object("id", class_id, Classroom, exceptions.ClassroomNotFound) + + +def get_classroom_from_token(class_token) -> Classroom: + """ + Gets a class without logging in. + + Args: + class_token (str): class token of the requested class + + Returns: + scratchattach.classroom.Classroom: An object representing the requested classroom + + Warning: + Any methods that require authentication will not work on the returned object. + + If you want to use these, get the user with :meth:`scratchattach.session.Session.connect_classroom` instead. + """ + warnings.warn( + "For methods that require authentication, use session.connect_classroom instead of get_classroom. If you want to remove this warning, use warnings.filterwarnings('ignore', category=ClassroomAuthenticationWarning). To ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use warnings.filterwarnings('ignore', category=GetAuthenticationWarning).", + exceptions.ClassroomAuthenticationWarning, + ) + return commons._get_object("classtoken", class_token, Classroom, exceptions.ClassroomNotFound) + + +def register_by_token( + class_id: int, + class_token: str, + username: str, + password: str, + birth_month: int, + birth_year: int, + gender: str, + country: str, + is_robot: bool = False, +) -> None: + data = { + "classroom_id": class_id, + "classroom_token": class_token, + "username": username, + "password": password, + "birth_month": birth_month, + "birth_year": birth_year, + "gender": gender, + "country": country, + "is_robot": is_robot, + } + response = requests.post( + "https://scratch.mit.edu/classes/register_new_student/", + data=data, + headers=commons.headers, + cookies={"scratchcsrftoken": "a"}, + ) + ret = response.json()[0] + if "username" in ret: + return + else: + raise exceptions.Unauthorized(f"Can't create account: {response.text}") diff --git a/scratchattach/async_api/site/cloud_activity.py b/scratchattach/async_api/site/cloud_activity.py new file mode 100644 index 00000000..39b31fad --- /dev/null +++ b/scratchattach/async_api/site/cloud_activity.py @@ -0,0 +1,122 @@ +from __future__ import annotations +import time +from typing import Union, TypeGuard, Optional +from dataclasses import dataclass, field +import warnings +from scratchattach.cloud import _base +from scratchattach.utils import exceptions +from scratchattach.site import user +from scratchattach.site import project as project_module +from ._base import BaseSiteComponent +from . import typed_dicts, session + + +@dataclass +class CloudActivity(BaseSiteComponent[Union[typed_dicts.CloudActivityDict, typed_dicts.CloudLogActivityDict]]): + """ + Represents a cloud activity (a cloud variable set / creation / deletion). + """ + + username: str = field(kw_only=True, default="") + "The user who caused the cloud event (the user who added / set / deleted the cloud variable)" + var: str = field(kw_only=True, default="") + "The name of the cloud variable that was updated (specified without the cloud emoji)" + name: str = field(kw_only=True, default="") + "The name of the cloud variable that was updated (specified without the cloud emoji)" + actual_var: str = field(kw_only=True, default="") + type: str = field(kw_only=True, default="set") + "The activity type" + timestamp: float = field(kw_only=True, default=0.0) + "Then timestamp of when the action was performed" + value: Union[float, int, str] = field(kw_only=True, default="0.0") + "If the cloud variable was set, then this attribute provides the value the cloud variable was set to" + cloud: _base.AnyCloud = field(kw_only=True, default_factory=lambda: _base.DummyCloud()) + "The cloud (as object inheriting from scratchattach.Cloud.BaseCloud) that the cloud activity corresponds to" + _session: Optional[session.Session] = field(kw_only=True, default=None) + + def update(self): + warnings.warn("CloudActivity objects can't be updated", exceptions.InvalidUpdateWarning) + return False + + def __eq__(self, activity2): + return ( + self.user == activity2.user + and self.type == activity2.type + and (self.timestamp == activity2.timestamp) + and (self.value == activity2.value) + and (self.name == activity2.name) + ) + + def _update_from_data(self, data: Union[typed_dicts.CloudActivityDict, typed_dicts.CloudLogActivityDict]) -> bool: + + def is_cloud_log_activity( + activity: Union[typed_dicts.CloudActivityDict, typed_dicts.CloudLogActivityDict], + ) -> TypeGuard[typed_dicts.CloudLogActivityDict]: + return "verb" in activity + + def is_cloud_activity( + activity: Union[typed_dicts.CloudActivityDict, typed_dicts.CloudLogActivityDict], + ) -> TypeGuard[typed_dicts.CloudActivityDict]: + return "method" in activity + + self.name = data["name"] + self.var = data["name"] + self.value = data["value"] + self.actual_var = data.get("variable_name") or self.var + if is_cloud_log_activity(data): + self.user = data["user"] + self.username = data["user"] + self.timestamp = data["timestamp"] + self.type = data["verb"].removesuffix("_var") + elif is_cloud_activity(data): + self.type = data["method"] + if "cloud" in data: + self.cloud = data["cloud"] + return True + + def a(self, **k): + pass + + def load_log_data(self): + if self.cloud is None: + print( + "Warning: There aren't cloud logs available for this cloud, therefore the user and exact timestamp can't be loaded" + ) + elif isinstance(self.cloud, _base.LogCloud): + logs = self.cloud.logs(filter_by_var_named=self.var, limit=100) + matching = list(filter(lambda x: x.value == self.value and x.timestamp <= self.timestamp, logs)) + if matching == []: + return False + activity = matching[0] + self.username = activity.username + self.user = activity.username + self.timestamp = activity.timestamp + return True + else: + print( + "Warning: There aren't cloud logs available for this cloud, therefore the user and exact timestamp can't be loaded" + ) + return False + + def actor(self): + """ + Returns the user that performed the cloud activity as scratchattach.user.User object + """ + if self.username is None: + return None + return self._make_linked_object("username", self.username, user.User, exceptions.UserNotFound) + + def project(self) -> Optional[project_module.Project]: + """ + Returns the project where the cloud activity was performed as scratchattach.project.Project object + """ + + def make_linked(cloud: _base.BaseCloud) -> project_module.Project: + return self._make_linked_object("id", cloud.project_id, project_module.Project, exceptions.ProjectNotFound) + + if self.cloud is None: + return None + cloud = self.cloud + if not isinstance(cloud, _base.BaseCloud): + return None + return make_linked(cloud) diff --git a/scratchattach/async_api/site/comment.py b/scratchattach/async_api/site/comment.py new file mode 100644 index 00000000..ab735c17 --- /dev/null +++ b/scratchattach/async_api/site/comment.py @@ -0,0 +1,207 @@ +"""Comment class""" + +from __future__ import annotations +import warnings +import html +from dataclasses import dataclass +from typing_extensions import assert_never +from typing import Union, Optional, Any +from enum import Enum, auto +from . import user, project, studio, session +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions + + +class CommentSource(Enum): + PROJECT = auto() + USER_PROFILE = auto() + STUDIO = auto() + UNKNOWN = auto() + + +@dataclass +class Comment(BaseSiteComponent): + """ + Represents a Scratch comment (on a profile, studio or project) + """ + + id: Optional[int | str] = None + source: CommentSource = CommentSource.UNKNOWN + source_id: Optional[int | str] = None + cached_replies: Optional[list[Comment]] = None + parent_id: Optional[int | str] = None + cached_parent_comment: Optional[Comment] = None + commentee_id: Optional[int] = None + content: Optional[str] = None + reply_count: Optional[int] = None + written_by_scratchteam: Optional[bool] = None + author_id: Optional[int] = None + author_name: Optional[str] = None + _session: Optional[session.Session] = None + + def __str__(self): + return self.text + + def update(self): + warnings.warn("Warning: Comment objects can't be updated") + return False + + def _update_from_data(self, data: dict[str, str | dict | Any]): + self.id = data["id"] + self.parent_id = data.get("parent_id") + self.commentee_id = data.get("commentee_id") + self.content = str(data["content"]) + self.datetime_created = data["datetime_created"] + author = data.get("author", {}) + self.author_name = author.get("username", self.author_name) + self.author_id = author.get("id", self.author_id) + self.written_by_scratchteam = author.get("scratchteam", self.written_by_scratchteam) + self.reply_count = data.get("reply_count", self.reply_count) + source: str = data.get("source") + if self.source is CommentSource.UNKNOWN: + self.source = { + "project": CommentSource.PROJECT, + "studio": CommentSource.STUDIO, + "profile": CommentSource.USER_PROFILE, + None: CommentSource.UNKNOWN, + }[source] + self.source_id = data.get("source_id", self.source_id) + + @property + def text(self) -> str: + """ + Parsed version of Comment.content. This removes any escape codes, e.g. ''' becomes ', an apostrophe + """ + if self.source is CommentSource.USER_PROFILE: + return self.content + return str(html.unescape(self.content)) + + def author(self) -> user.User: + return self._make_linked_object("username", self.author_name, user.User, exceptions.UserNotFound) + + def place(self) -> user.User | studio.Studio | project.Project: + """ + Returns the place (the project, profile or studio) where the comment was posted as Project, User, or Studio object. + + If the place can't be traced back, None is returned. + """ + if self.source == CommentSource.USER_PROFILE: + return self._make_linked_object("username", self.source_id, user.User, exceptions.UserNotFound) + elif self.source == CommentSource.STUDIO: + return self._make_linked_object("id", self.source_id, studio.Studio, exceptions.UserNotFound) + elif self.source == CommentSource.PROJECT: + return self._make_linked_object("id", self.source_id, project.Project, exceptions.UserNotFound) + else: + assert_never(self.source) + + def parent_comment(self) -> Comment | None: + if self.parent_id is None: + return None + if self.cached_parent_comment is not None: + return self.cached_parent_comment + if self.source == "profile": + self.cached_parent_comment = user.User(username=self.source_id, _session=self._session).comment_by_id( + self.parent_id + ) + elif self.source == "project": + p = project.Project(id=self.source_id, _session=self._session) + p.update() + self.cached_parent_comment = p.comment_by_id(self.parent_id) + elif self.source == "studio": + self.cached_parent_comment = studio.Studio(id=self.source_id, _session=self._session).comment_by_id(self.parent_id) + return self.cached_parent_comment + + def replies(self, *, use_cache: bool = True, limit=40, offset=0): + """ + Keyword Arguments: + use_cache (bool): Returns the replies cached on the first reply fetch. This makes it SIGNIFICANTLY faster for profile comments. Warning: For profile comments, the replies are retrieved and cached on object creation. + """ + if self.cached_replies is None or not use_cache: + if self.source == CommentSource.USER_PROFILE: + self.cached_replies = ( + user.User(username=self.source_id, _session=self._session) + .comment_by_id(self.id) + .cached_replies[offset : offset + limit] + ) + elif self.source == CommentSource.PROJECT: + p = project.Project(id=self.source_id, _session=self._session) + p.update() + self.cached_replies = p.comment_replies(comment_id=self.id, limit=limit, offset=offset) + elif self.source == CommentSource.STUDIO: + self.cached_replies = studio.Studio(id=self.source_id, _session=self._session).comment_replies( + comment_id=self.id, limit=limit, offset=offset + ) + return self.cached_replies + + def reply(self, content, *, commentee_id=None): + """ + Posts a reply comment to the comment. + + Warning: + Scratch only shows comments replying to top-level comments, and all replies to replies are actually replies to top-level comments in the API. + + Therefore, if this comment is a reply, this method will not reply to the comment itself but to the corresponding top-level comment. + + Args: + content (str): Comment content to post. + + Keyword args: + commentee_id (None or str): If set to None (default), it will automatically fill out the commentee ID with the user ID of the parent comment author. Set it to "" to mention no user. + + + Returns: + scratchattach.Comment: The created comment. + :param content: Content of the comment to send + :param commentee_id: ID of user to reply to + """ + self._assert_auth() + parent_id = str(self.id) + if self.parent_id is not None: + parent_id = str(self.parent_id) + if commentee_id is None: + if self.author_id: + commentee_id = self.author_id + else: + commentee_id = "" + if self.source == CommentSource.USER_PROFILE: + return user.User(username=self.source_id, _session=self._session).reply_comment( + content, parent_id=str(parent_id), commentee_id=commentee_id + ) + if self.source == CommentSource.PROJECT: + p = project.Project(id=self.source_id, _session=self._session) + p.update() + return p.reply_comment(content, parent_id=str(parent_id), commentee_id=commentee_id) + if self.source == CommentSource.STUDIO: + return studio.Studio(id=self.source_id, _session=self._session).reply_comment( + content, parent_id=str(parent_id), commentee_id=commentee_id + ) + raise ValueError(f"Unknown source: {self.source}") + + def delete(self): + """ + Deletes the comment. + """ + self._assert_auth() + if self.source == CommentSource.USER_PROFILE: + return user.User(username=self.source_id, _session=self._session).delete_comment(comment_id=self.id) + elif self.source == CommentSource.PROJECT: + p = project.Project(id=self.source_id, _session=self._session) + p.update() + return p.delete_comment(comment_id=self.id) + elif self.source == CommentSource.STUDIO: + return studio.Studio(id=self.source_id, _session=self._session).delete_comment(comment_id=self.id) + return None + + def report(self): + """ + Reports the comment to the Scratch team. + """ + self._assert_auth() + if self.source == CommentSource.USER_PROFILE: + user.User(username=self.source_id, _session=self._session).report_comment(comment_id=self.id) + elif self.source == CommentSource.PROJECT: + p = project.Project(id=self.source_id, _session=self._session) + p.update() + p.report_comment(comment_id=self.id) + elif self.source == CommentSource.STUDIO: + studio.Studio(id=self.source_id, _session=self._session).report_comment(comment_id=self.id) diff --git a/scratchattach/async_api/site/forum.py b/scratchattach/async_api/site/forum.py new file mode 100644 index 00000000..61bb25d1 --- /dev/null +++ b/scratchattach/async_api/site/forum.py @@ -0,0 +1,416 @@ +"""ForumTopic and ForumPost classes""" + +from __future__ import annotations +import warnings +from dataclasses import dataclass, field +from typing import Optional, Any +from urllib.parse import urlparse, parse_qs +import xml.etree.ElementTree as ET +from bs4 import BeautifulSoup, Tag +from . import user +from . import session as module_session +from scratchattach.utils.commons import headers +from scratchattach.utils import exceptions, commons +from ._base import BaseSiteComponent +from scratchattach.utils.requests import requests + + +@dataclass +class ForumTopic(BaseSiteComponent): + """ + Represents a Scratch forum topic. + + Attributes: + + :.id: + + :.title: + + :.category_name: + + :.last_updated: + + Attributes only available if the object was created using scratchattach.get_topic_list or scratchattach.Session.connect_topic_list: + + :.reply_count: + + :.view_count: + + :.update(): Updates the attributes + """ + + id: int + title: str + category_name: Optional[str] = None + last_updated: Optional[str] = None + _session: Optional[module_session.Session] = field(default=None) + reply_count: Optional[int] = field(default=None) + view_count: Optional[int] = field(default=None) + + def __str__(self): + return f"-F {self.title} ({self.id})" + + def __post_init__(self): + self.update_function = requests.get + self.update_api = f"https://scratch.mit.edu/discuss/feeds/topic/{self.id}/" + if self._session is None: + self._headers = headers + self._cookies = {} + else: + self._headers = self._session.get_headers() + self._cookies = self._session.get_cookies() + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def update(self): + response = self.update_function(self.update_api, headers=self._headers, cookies=self._cookies, timeout=20) + if "429" in str(response): + return "429" + if response.status_code == 200: + try: + root = ET.fromstring(response.text) + namespace = {"atom": "http://www.w3.org/2005/Atom"} + title = root.findtext("atom:title", namespaces=namespace).replace("Latest posts on ", "") + category_name = ( + root.findall(".//atom:entry", namespaces=namespace)[0] + .findtext(".//atom:title", namespaces=namespace) + .split(" :: ")[1] + ) + last_updated = root.findtext("atom:updated", namespaces=namespace) + except Exception as e: + raise exceptions.ScrapeError(str(e)) + else: + raise exceptions.ForumContentNotFound + self.title = title + self.category_name = category_name + self.last_updated = last_updated + return True + + @classmethod + def from_id(cls, __id: int, session: module_session.Session, update: bool = False): + new = cls(id=__id, _session=session, title="", last_updated="", category_name="") + if update: + new.update() + return new + + def _update_from_data(self, data: dict[str, Any]): + self.__dict__.update(data) + + def posts(self, *, page=1, order="oldest") -> list[ForumPost]: + """ + Args: + page (int): The page of the forum topic that should be returned. First page is at index 1. + + Returns: + list: A list containing the posts from the specified page of the forum topic + """ + if order != "oldest": + warnings.warn("Warning: All post orders except for 'oldest' are deprecated and no longer work") + posts = [] + try: + url = f"https://scratch.mit.edu/discuss/topic/{self.id}/?page={page}" + response = requests.get(url, headers=headers, cookies=self._cookies) + except Exception as e: + raise exceptions.FetchError(str(e)) + try: + soup = BeautifulSoup(response.content, "html.parser") + soup_elm = soup.find("div", class_="djangobb") + assert isinstance(soup_elm, Tag) + try: + pagination_div = soup_elm.find("div", class_="pagination") + assert isinstance(pagination_div, Tag) + num_pages = int(pagination_div.find_all("a", class_="page")[-1].text) + except Exception: + num_pages = 1 + try: + topic_category = "" + breadcrumb_ul = soup_elm.find_all("ul")[1] + if breadcrumb_ul: + assert isinstance(breadcrumb_ul, Tag) + link = breadcrumb_ul.find_all("a")[1] + topic_category = link.text.strip() + except Exception as e: + warnings.warn(f"Warning: Couldn't scrape topic category for topic {self.id} - {e}") + topic_category = "" + post_htmls = soup.find_all("div", class_="blockpost") + for raw_post in post_htmls: + if not isinstance(raw_post, Tag): + continue + post = ForumPost( + id=int(str(raw_post["id"]).replace("p", "")), + topic_id=self.id, + _session=self._session, + topic_category=topic_category, + topic_num_pages=num_pages, + ) + post.update_from_html(raw_post) + posts.append(post) + except Exception as e: + raise exceptions.ScrapeError() from e + return posts + + def first_post(self): + """ + Returns: + scratchattach.forum.ForumPost: An object representing the first topic post + """ + posts = self.posts(page=1) + if len(posts) > 0: + return posts[0] + + +@dataclass +class ForumPost(BaseSiteComponent): + """ + Represents a Scratch forum post. + + Attributes: + + :.id: + + :.author_name: The name of the user who created this post + + :.author_avatar_url: + + :.posted: The date the post was made + + :.topic_id: The id of the topic this post is in + + :.topic_name: The name of the topic the post is in + + :.topic_category: The name of the category the post topic is in + + :.topic_num_pages: The number of pages the post topic has + + :.deleted: Whether the post was deleted (always False because deleted posts can't be retrieved anymore) + + :.html_content: Returns the content as HTML + + :.content: Returns the content as text + + :.post_index: The index that the post has in the topic + + :.update(): Updates the attributes + """ + + id: int = field(default=0) + topic_id: int = field(default=0) + topic_name: str = field(default="") + topic_category: str = field(default="") + topic_num_pages: int = field(default=0) + author_name: str = field(default="") + author_avatar_url: str = field(default="") + posted: str = field(default="") + deleted: bool = field(default=False) + html_content: str = field(default="") + content: str = field(default="") + post_index: int = field(default=0) + _session: Optional[module_session.Session] = field(default=None) + + def __post_init__(self): + self.update_api = "" + if self._session is None: + self._headers = headers + self._cookies = {} + else: + self._headers = self._session.get_headers() + self._cookies = self._session.get_cookies() + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def update_function(self, *args, **kwargs): + raise TypeError("Forum posts cannot be updated like this") + + def update(self): + """ + Updates the attributes of the ForumPost object. + As there is no API for retrieving a single post anymore, this requires reloading the forum page. + """ + page = 1 + posts = ForumTopic.from_id(self.topic_id, session=self._session).posts(page=1) + while posts != []: + matching = list(filter(lambda x: int(x.id) == int(self.id), posts)) + if len(matching) > 0: + this = matching[0] + break + page += 1 + posts = ForumTopic.from_id(self.topic_id, session=self._session).posts(page=page) + else: + return False + self._update_from_data(vars(this)) + + def _update_from_data(self, data: dict[str, Any]): + self.__dict__.update(data) + return True + + def update_from_html(self, soup_html: Tag): + return self._update_from_html(soup_html) + + def _update_from_html(self, soup_html: Tag): + post_index_elm = soup_html.find("span", class_="conr") + assert isinstance(post_index_elm, Tag) + id_attr = soup_html["id"] + assert isinstance(id_attr, str) + posted_elm = soup_html.find("a", href=True) + assert isinstance(posted_elm, Tag) + content_elm = soup_html.find("div", class_="post_body_html") + assert isinstance(content_elm, Tag) + author_name_elm = soup_html.select_one("dl dt a") + assert isinstance(author_name_elm, Tag) + topic_name_elm = soup_html.find("h3") + assert isinstance(topic_name_elm, Tag) + self.post_index = int(post_index_elm.text.strip("#")) + self.id = int(id_attr.replace("p", "")) + self.posted = posted_elm.text.strip() + self.content = content_elm.text.strip() + self.html_content = str(soup_html.find("div", class_="post_body_html")) + self.author_name = author_name_elm.text.strip() + self.author_avatar_url = str(author_name_elm["href"]) + self.topic_name = topic_name_elm.text.strip() + return True + + def topic(self): + """ + Returns: + scratchattach.forum.ForumTopic: An object representing the forum topic this post is in. + """ + return self._make_linked_object("id", self.topic_id, ForumTopic, exceptions.ForumContentNotFound) + + def ocular_reactions(self): + return requests.get(f"https://my-ocular.jeffalo.net/api/reactions/{self.id}", timeout=10).json() + + def author(self): + """ + Returns: + scratchattach.user.User: An object representing the user who created this forum post. + """ + return self._make_linked_object("username", self.author_name, user.User, exceptions.UserNotFound) + + def edit(self, new_content: str): + """ + Changes the content of the forum post. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_post` or through another method that requires authentication. You must own the forum post. + + Args: + new_content (str): The text that the forum post will be set to. + """ + self._assert_auth() + cookies = dict(self._cookies) + cookies["accept"] = ( + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" + ) + cookies["Content-Type"] = "application/x-www-form-urlencoded" + r = requests.post( + f"https://scratch.mit.edu/discuss/post/{self.id}/edit/", + headers={ + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "accept-language": "de,en;q=0.9", + "cache-control": "max-age=0", + "content-type": "application/x-www-form-urlencoded", + "sec-ch-ua": '" Not A;Brand";v="99", "Chromium";v="101", "Google Chrome";v="101"', + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": '"Windows"', + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "same-origin", + "sec-fetch-user": "?1", + "upgrade-insecure-requests": "1", + "Referer": f"https://scratch.mit.edu/discuss/post/{self.id}/edit/", + "x-csrftoken": "a", + }, + cookies=cookies, + json=f"csrfmiddlewaretoken=a&body={new_content}&", + timeout=10, + ) + + +def get_topic(topic_id) -> ForumTopic: + """ + Gets a forum topic without logging in. Data received from Scratch's RSS feed XML API. + + Args: + topic_id (int): ID of the requested forum topic + + Returns: + scratchattach.forum.ForumTopic: An object representing the requested forum topic + + Warning: + Scratch's API uses very heavy caching for logged out users, therefore the returned data will not be up to date. + + Any methods that require authentication will not work on the returned object. + + If you need up-to-date data or want to use methods that require authentication, create the object with :meth:`scratchattach.session.Session.connect_topic` instead. + """ + return commons._get_object("id", topic_id, ForumTopic, exceptions.ForumContentNotFound) + + +def get_topic_list(category_id, *, page=1): + """ + Gets the topics from a forum category without logging in. Data web-scraped from Scratch's forums UI. + + Args: + category_id (str): ID of the forum category + + Keyword Arguments: + page (str): Page of the category topics that should be returned + + Returns: + list: A list containing the forum topics from the specified category + + Warning: + Scratch's API uses very heavy caching for logged out users, therefore the returned data will not be up to date. + + Any methods that require authentication will not work on the returned objects. + + If you need up-to-date data or want to use methods that require authentication, get the forum topics with :meth:`scratchattach.session.Session.connect_topic_list` instead. + """ + try: + response = requests.get(f"https://scratch.mit.edu/discuss/{category_id}/?page={page}") + soup = BeautifulSoup(response.content, "html.parser") + except Exception as e: + raise exceptions.FetchError(str(e)) + try: + category_name = soup.find("h4").find("span").get_text() + except Exception as e: + raise exceptions.BadRequest("Invalid category id") + try: + topics = soup.find_all("tr") + topics.pop(0) + return_topics = [] + for topic in topics: + title_link = topic.find("a") + title = title_link.text.strip() + topic_id = title_link["href"].split("/")[-2] + columns = topic.find_all("td") + columns = [column.text for column in columns] + if len(columns) == 1: + continue + last_updated = columns[3].split(" ")[0] + " " + columns[3].split(" ")[1] + return_topics.append( + ForumTopic( + id=int(topic_id), + title=title, + category_name=category_name, + last_updated=last_updated, + reply_count=int(columns[1]), + view_count=int(columns[2]), + ) + ) + return return_topics + except Exception as e: + raise exceptions.ScrapeError(str(e)) + + +def youtube_link_to_scratch(link: str): + """ + Converts a YouTube url (in multiple formats) like https://youtu.be/1JTgg4WVAX8?si=fIEskaEaOIRZyTAz + to a link like https://scratch.mit.edu/discuss/youtube/1JTgg4WVAX8 + """ + url_parse = urlparse(link) + query_parse = parse_qs(url_parse.query) + if "v" in query_parse: + video_id = query_parse["v"][0] + else: + video_id = url_parse.path.split("/")[-1] + return f"https://scratch.mit.edu/discuss/youtube/{video_id}" diff --git a/scratchattach/async_api/site/project.py b/scratchattach/async_api/site/project.py new file mode 100644 index 00000000..a2ca7f2d --- /dev/null +++ b/scratchattach/async_api/site/project.py @@ -0,0 +1,791 @@ +"""Project and PartialProject classes""" + +from __future__ import annotations +import json +import random +import base64 +import time +import warnings +import zipfile +from io import BytesIO +from typing import Callable, Union +from dataclasses import dataclass, field +from typing import Any, Optional +from typing_extensions import deprecated +from scratchattach.site.typed_dicts import ProjectDict +from . import user, comment, studio, session +from scratchattach.utils import exceptions +from scratchattach.utils import commons +from scratchattach.utils.commons import empty_project_json, headers +from ._base import BaseSiteComponent +from scratchattach import editor +from scratchattach.utils.requests import requests + +CREATE_PROJECT_USES: list[float] = [] + + +@dataclass +class PartialProject(BaseSiteComponent): + """ + Represents an unshared Scratch project that can't be accessed. + """ + + id: Union[str, int] = field(kw_only=True, default=0) + "The project id" + url: str = field(kw_only=True, default="") + "The project url" + title: str = field(kw_only=True, default="") + author_name: str = field(kw_only=True, default="") + "The username of the author" + comments_allowed: bool = field(kw_only=True, default=False) + "whether comments are enabled" + comment_count: int = 0 + "The number of comments on the project (this may be unreliable)" + instructions: str = field(kw_only=True, default="") + notes: str = field(kw_only=True, default="") + "The 'Notes and Credits' section" + created: str = field(kw_only=True, default="") + "The date of the project creation" + last_modified: str = field(kw_only=True, default="") + "The date when the project was modified the last time" + share_date: str = field(kw_only=True, default="") + thumbnail_url: str = field(kw_only=True, default="") + remix_parent: Optional[Union[str, int]] = field(kw_only=True, default="") + parent_title: Optional[str] = field(kw_only=True, default=None) + remix_root: Optional[Union[str, int]] = field(kw_only=True, default="") + loves: int = field(kw_only=True, default=0) + "The project's love count" + favorites: int = field(kw_only=True, default=0) + "The project's favorite count" + remix_count: int = field(kw_only=True, default=0) + "The number of remixes" + views: int = field(kw_only=True, default=0) + "The view count" + project_token: Optional[str] = field(kw_only=True, default=None) + "The project token (required to access the project json)" + _moderation_status: Optional[str] = field(kw_only=True, default=None) + _session: Optional[session.Session] = field(kw_only=True, default=None) + + def __str__(self): + return f"Unshared project with id {self.id}" + + def __post_init__(self) -> None: + self.update_function: Callable = requests.get + self.update_api = f"https://api.scratch.mit.edu/projects/{self.id}" + if self._session is None: + self._headers = headers + self._cookies = {} + else: + self._headers = self._session.get_headers() + self._cookies = self._session.get_cookies() + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def _update_from_data(self, data: ProjectDict): + self.id = int(data.get("id", self.id)) + self.url = f"https://scratch.mit.edu/projects/{self.id}" + if author := data.get("author"): + self.author_name = author.get("username", self.author_name) + self.author_name = data.get("username", self.author_name) + self.comments_allowed = data.get("comments_allowed", self.comments_allowed) + self.instructions = data.get("instructions", self.instructions) + self.notes = data.get("description", self.notes) + if history := data.get("history"): + self.created = history.get("created", self.created) + self.last_modified = history.get("modified", self.last_modified) + self.share_date = history.get("shared", self.share_date) + self.thumbnail_url = data.get("image", self.thumbnail_url) + remix_data = data.get("remix", {}) + self.remix_parent = remix_data.get("parent") + self.remix_root = remix_data.get("root") + if stats := data.get("stats"): + self.favorites = stats.get("favorites", self.favorites) + self.loves = stats.get("loves", self.loves) + self.remix_count = stats.get("remixes", self.remix_count) + self.views = stats.get("views", self.views) + self.title = data.get("title", self.title) + self.project_token = data.get("project_token", None) + return "code" not in data + + def __rich__(self): + from rich.panel import Panel + from rich.table import Table + from rich import box + from rich.markup import escape + + url = f"[link={self.url}]{self.title}[/]" + ret = Table.grid(expand=True) + ret.add_column(ratio=1) + ret.add_column(ratio=3) + info = Table(box=box.SIMPLE) + info.add_column(url, overflow="fold") + info.add_column(f"#{self.id}", overflow="fold") + info.add_row("By", self.author_name) + info.add_row("Created", escape(self.created)) + info.add_row("Shared", escape(self.share_date)) + info.add_row("Modified", escape(self.last_modified)) + info.add_row("Comments allowed", escape(str(self.comments_allowed))) + info.add_row("Loves", str(self.loves)) + info.add_row("Faves", str(self.favorites)) + info.add_row("Remixes", str(self.remix_count)) + info.add_row("Views", str(self.views)) + desc = Table(box=box.SIMPLE) + desc.add_row("Instructions", escape(self.instructions)) + desc.add_row("Notes & Credits", escape(self.notes)) + ret.add_row(Panel(info, title=url), Panel(desc, title="Description")) + return ret + + @property + def embed_url(self): + """ + Returns: + the url of the embed of the project + """ + return f"{self.url}/embed" + + def remixes(self, *, limit=40, offset=0) -> list[Project]: + """ + Returns: + list: A list containing the remixes of the project, each project is represented by a Project object. + """ + response = commons.api_iterative(f"https://api.scratch.mit.edu/projects/{self.id}/remixes", limit=limit, offset=offset) + return commons.parse_object_list(response, Project, self._session) + + def is_shared(self): + """ + Returns: + boolean: Returns whether the project is currently shared + """ + try: + p = get_project(self.id) + return isinstance(p, Project) + except exceptions.ProjectNotFound: + return False + + def raw_json_or_empty(self) -> dict[str, Any]: + return empty_project_json + + def create_remix(self, *, title=None, project_json=None) -> Project: + """ + Creates a project on the Scratch website. + + Warning: + Don't spam this method - it WILL get you banned from Scratch. + To prevent accidental spam, a rate limit (5 projects per minute) is implemented for this function. + """ + session = self._assert_auth() + if title is None: + if "title" in self.__dict__: + title = self.title + " remix" + else: + title = " remix" + if project_json is None: + project_json = self.raw_json_or_empty() + if len(CREATE_PROJECT_USES) < 5: + CREATE_PROJECT_USES.insert(0, time.time()) + else: + if CREATE_PROJECT_USES[-1] < time.time() - 300: + CREATE_PROJECT_USES.pop() + else: + raise exceptions.BadRequest( + "Rate limit for remixing Scratch projects exceeded.\nThis rate limit is enforced by scratchattach, not by the Scratch API.\nFor security reasons, it cannot be turned off.\n\nDon't spam-create projects, it WILL get you banned." + ) + CREATE_PROJECT_USES.insert(0, time.time()) + params = {"is_remix": "1", "original_id": self.id, "title": title} + response = requests.post( + "https://projects.scratch.mit.edu/", params=params, cookies=self._cookies, headers=self._headers, json=project_json + ).json() + _project = session.connect_project(response["content-name"]) + _project.parent_title = base64.b64decode(response["content-title"]).decode("utf-8").split(" remix")[0] + return _project + + def load_description(self): + """ + Gets the instructions of the unshared project. Requires authentication. + + Warning: + It's unclear if Scratch allows using this method. This method will create a remix of the unshared project using your account. + """ + self._assert_auth() + new_project = self.create_remix(project_json=empty_project_json) + self.instructions = new_project.instructions + self.title = new_project.parent_title or "" + + +@dataclass +class Project(PartialProject): + """ + Represents a Scratch project. + """ + + def __repr__(self): + return f"-P {self.id} ({self.title})" + + def __str__(self): + return repr(self) + + @property + def thumbnail(self) -> bytes: + with requests.no_error_handling(): + return requests.get(self.thumbnail_url).content + + def _assert_permission(self): + session = self._assert_auth() + if session.username != self.author_name: + raise exceptions.Unauthorized("You need to be authenticated as the profile owner to do this.") + + def load_description(self): + self.update() + + def download(self, *, filename=None, dir="."): + """ + Downloads the project json to the given directory. + + Args: + filename (str): The name that will be given to the downloaded file. + dir (str): The path of the directory the file will be saved in. + """ + try: + if filename is None: + filename = str(self.id) + if not (dir.endswith("/") or dir.endswith("\\")): + dir += "/" + self.update() + response = requests.get(f"https://projects.scratch.mit.edu/{self.id}?token={self.project_token}", timeout=10) + filename = filename.removesuffix(".sb3") + with open(f"{dir}{filename}.sb3", "wb") as f: + f.write(response.content) + except Exception as exc: + raise exceptions.FetchError("Method only works for projects created with Scratch 3") from exc + + @deprecated("Use raw_json instead") + def get_json(self) -> str: + """ + Downloads the project json and returns it as a string + """ + try: + self.update() + response = requests.get(f"https://projects.scratch.mit.edu/{self.id}?token={self.project_token}", timeout=10) + return response.text + except Exception as exc: + raise exceptions.FetchError("Method only works for projects created with Scratch 3") from exc + + def body(self) -> editor.Project: + """ + Method only works for project created with Scratch 3. + + Returns: + scratchattach.editor.Project: The contents of the project as editor Project object + """ + raw_json = self.raw_json() + return editor.Project.from_json(raw_json) + + def raw_json(self): + """ + Method only works for project created with Scratch 3. + + Returns: + dict: The raw project JSON as decoded Python dictionary + """ + try: + self.update() + except Exception as e: + raise exceptions.FetchError(f"You're not authorized for accessing {self}.\nException: {e}") + with requests.no_error_handling(): + resp = requests.get(f"https://projects.scratch.mit.edu/{self.id}?token={self.project_token}", timeout=10) + try: + return resp.json() + except json.JSONDecodeError: + with zipfile.ZipFile(BytesIO(resp.content)) as zipf: + return json.load(zipf.open("project.json")) + + def raw_json_or_empty(self): + return self.raw_json() + + def creator_agent(self): + """ + Method only works for project created with Scratch 3. + + Returns: + str: The user agent of the browser that this project was saved with. + """ + return self.raw_json()["meta"]["agent"] + + def set_body(self, project_body: editor.Project): + """ + Sets the project's contents You can use this to upload projects to the Scratch website. + Returns a dict with Scratch's raw JSON API response. + + Args: + project_body (scratchattach.ProjectBody): A ProjectBody object containing the contents of the project + """ + self._assert_permission() + return self.set_json(project_body.to_json()) + + def set_json(self, json_data): + """ + Sets the project json. You can use this to upload projects to the Scratch website. + Returns a dict with Scratch's raw JSON API response. + + Args: + json_data (dict or JSON): The new project JSON as encoded JSON object or as dict + """ + self._assert_permission() + if not isinstance(json_data, dict): + json_data = json.loads(json_data) + return requests.put( + f"https://projects.scratch.mit.edu/{self.id}", headers=self._headers, cookies=self._cookies, json=json_data + ).json() + + def upload_json_from(self, project_id: int | str): + """ + Uploads the project json from the project with the given id to the project represented by this Project object + """ + self._assert_auth() + other_project = self._session.connect_project(project_id) + self.set_json(other_project.raw_json()) + + def author(self) -> user.User: + """ + Returns: + scratchattach.user.User: An object representing the Scratch user who created this project. + """ + return self._make_linked_object("username", self.author_name, user.User, exceptions.UserNotFound) + + def studios(self, *, limit=40, offset=0): + """ + Returns: + list: A list containing the studios this project is in, each studio is represented by a Studio object. + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/studios", + limit=limit, + offset=offset, + add_params=f"&cachebust={random.randint(0, 9999)}", + ) + return commons.parse_object_list(response, studio.Studio, self._session) + + def comments(self, *, limit=40, offset=0) -> list["comment.Comment"]: + """ + Returns the comments posted on the project (except for replies. To get replies use :meth:`scratchattach.project.Project.comment_replies`). + + Keyword Arguments: + page: The page of the comments that should be returned. + limit: Max. amount of returned comments. + + Returns: + list: A list containing the requested comments as Comment objects. + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/comments/", + limit=limit, + offset=offset, + add_params=f"&cachebust={random.randint(0, 9999)}", + _headers=self._headers, + cookies=self._cookies, + ) + for i in response: + i["source"] = "project" + i["source_id"] = self.id + return commons.parse_object_list(response, comment.Comment, self._session) + + def comment_replies(self, *, comment_id, limit=40, offset=0): + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/comments/{comment_id}/replies/", + limit=limit, + offset=offset, + add_params=f"&cachebust={random.randint(0, 9999)}", + _headers=self._headers, + cookies=self._cookies, + ) + for x in response: + x["parent_id"] = comment_id + x["source"] = "project" + x["source_id"] = self.id + return commons.parse_object_list(response, comment.Comment, self._session) + + def comment_by_id(self, comment_id): + """ + Returns: + scratchattach.comments.Comment: A Comment object representing the requested comment. + """ + data = requests.get( + f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/comments/{comment_id}", + headers=self._headers, + cookies=self._cookies, + ).json() + if data is None or data.get("code") == "NotFound": + raise exceptions.CommentNotFound(f"Cannot find comment #{comment_id} on -P {self.id} by -U {self.author_name}") + _comment = comment.Comment( + id=data["id"], _session=self._session, source=comment.CommentSource.PROJECT, source_id=self.id + ) + _comment._update_from_data(data) + return _comment + + def love(self): + """ + Posts a love on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + session = self._assert_auth() + r = requests.post( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/loves/user/{session.username}", + headers=self._headers, + cookies=self._cookies, + ).json() + if "userLove" in r: + if r["userLove"] is False: + self.love() + else: + raise exceptions.APIError(str(r)) + + def unlove(self): + """ + Removes the love from this project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + session = self._assert_auth() + r = requests.delete( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/loves/user/{session.username}", + headers=self._headers, + cookies=self._cookies, + ).json() + if "userLove" in r: + if r["userLove"] is True: + self.unlove() + else: + raise exceptions.APIError(str(r)) + + def favorite(self): + """ + Posts a favorite on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + session = self._assert_auth() + r = requests.post( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/favorites/user/{session.username}", + headers=self._headers, + cookies=self._cookies, + ).json() + if "userFavorite" in r: + if r["userFavorite"] is False: + self.favorite() + else: + raise exceptions.APIError(str(r)) + + def unfavorite(self): + """ + Removes the favorite from this project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + session = self._assert_auth() + r = requests.delete( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/favorites/user/{session.username}", + headers=self._headers, + cookies=self._cookies, + ).json() + if "userFavorite" in r: + if r["userFavorite"] is True: + self.unfavorite() + else: + raise exceptions.APIError(str(r)) + + def post_view(self): + """ + Increases the project's view counter by 1. Doesn't require a login. + """ + requests.post(f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/views/", headers=headers) + + def set_fields(self, fields_dict, *, use_site_api=False): + """ + Sets fields. By default, ueses the api.scratch.mit.edu/projects/xxx/ PUT API. + + Keyword Arguments: + use_site_api (bool): + When enabled, the fields are set using the scratch.mit.edu/site-api API. + This function allows setting more fields than Project.set_fields. + For example, you can also share / unshare the project by setting the "shared" field. + According to the Scratch team, this API is deprecated. As of 2024 it's still fully functional though. + """ + self._assert_permission() + if use_site_api: + r = requests.put( + f"https://scratch.mit.edu/site-api/projects/all/{self.id}", + headers=self._headers, + cookies=self._cookies, + json=fields_dict, + ).json() + else: + r = requests.put( + f"https://api.scratch.mit.edu/projects/{self.id}", + headers=self._headers, + cookies=self._cookies, + json=fields_dict, + ).json() + return self._update_from_data(r) + + def turn_off_commenting(self): + """ + Disables commenting on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + data = {"comments_allowed": False} + self.set_fields(data) + + def turn_on_commenting(self): + """ + Enables commenting on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + data = {"comments_allowed": True} + self.set_fields(data) + + def toggle_commenting(self): + """ + Switches commenting on / off on the project (If comments are on, they will be turned off, else they will be turned on). You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + data = {"comments_allowed": not self.comments_allowed} + self.set_fields(data) + + def share(self): + """ + Shares the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self._assert_permission() + requests.put( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/share/", headers=self._json_headers, cookies=self._cookies + ) + + def unshare(self): + """ + Unshares the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self._assert_permission() + requests.put( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/unshare/", headers=self._json_headers, cookies=self._cookies + ) + + ' doesn\'t work. the API\'s response is valid (no errors), but the fields don\'t change\n def move_to_trash(self):\n """\n Moves the project to trash folder. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project`\n """\n self.set_fields({"id":int(self.id), "visibility": "trshbyusr", "isPublished" : False}, use_site_api=True)' + + def set_thumbnail(self, *, file): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self._assert_permission() + with open(file, "rb") as f: + thumbnail = f.read() + requests.post( + f"https://scratch.mit.edu/internalapi/project/thumbnail/{self.id}/set/", + data=thumbnail, + headers=self._headers, + cookies=self._cookies, + ) + + def delete_comment(self, *, comment_id): + """ + Deletes a comment by its ID. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + comment_id: The id of the comment that should be deleted + """ + self._assert_permission() + return requests.delete( + f"https://api.scratch.mit.edu/proxy/comments/project/{self.id}/comment/{comment_id}/", + headers=self._headers, + cookies=self._cookies, + ) + + def report_comment(self, *, comment_id): + """ + Reports a comment by its ID to the Scratch team. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + comment_id: The id of the comment that should be reported + """ + self._assert_auth() + return requests.delete( + f"https://api.scratch.mit.edu/proxy/comments/project/{self.id}/comment/{comment_id}/report", + headers=self._headers, + cookies=self._cookies, + ) + + def post_comment(self, content, *, parent_id="", commentee_id=""): + """ + Posts a comment on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + content: Content of the comment that should be posted + + Keyword Arguments: + parent_id: ID of the comment you want to reply to. If you don't want to mention a user, don't put the argument. + commentee_id: ID of the user that will be mentioned in your comment and will receive a message about your comment. If you don't want to mention a user, don't put the argument. + + Returns: + scratchattach.comments.Comment: Comment object representing the posted comment. + """ + self._assert_auth() + data = {"commentee_id": commentee_id, "content": str(content), "parent_id": parent_id} + r = json.loads( + requests.post( + f"https://api.scratch.mit.edu/proxy/comments/project/{self.id}/", + headers=self._json_headers | {"referer": "https://scratch.mit.edu/projects/" + str(self.id) + "/"}, + cookies=self._cookies, + data=json.dumps(data), + ).text + ) + if "id" not in r: + raise exceptions.CommentPostFailure(r) + _comment = comment.Comment(id=r["id"], _session=self._session, source=comment.CommentSource.PROJECT, source_id=self.id) + _comment._update_from_data(r) + return _comment + + def reply_comment(self, content, *, parent_id, commentee_id=""): + """ + Posts a reply to a comment on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + content: Content of the comment that should be posted + + Warning: + Only replies to top-level comments are shown on the Scratch website. Replies to replies are actually replies to the corresponding top-level comment in the API. + + Therefore, parent_id should be the comment id of a top level comment. + + Keyword Arguments: + parent_id: ID of the comment you want to reply to + commentee_id: ID of the user you are replying to + """ + return self.post_comment(content, parent_id=parent_id, commentee_id=commentee_id) + + def set_title(self, text): + """ + Changes the projects title. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self.set_fields({"title": text}) + + def set_instructions(self, text): + """ + Changes the projects instructions. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self.set_fields({"instructions": text}) + + def set_notes(self, text): + """ + Changes the projects notes and credits. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self.set_fields({"description": text}) + + @deprecated("Deprecated because ScratchDB is down indefinitely.") + def ranks(self): + """ + Gets information about the project's ranks. Fetched from ScratchDB. + + Warning: + This method is deprecated because ScratchDB is down indefinitely. + + Returns: + dict: A dict containing the project's ranks. If the ranks aren't available, all values will be -1. + """ + return requests.get(f"https://scratchdb.lefty.one/v3/project/info/{self.id}").json()["statistics"]["ranks"] + + def moderation_status(self, *, reload: bool = False): + """ + Gets information about the project's moderation status. Fetched from jeffalo's API. + + Returns: + str: The moderation status of the project. + + These moderation statuses exist: + + safe: The project was reviewed by the Scratch team and was considered safe for everyone. + + notsafe: The project was reviewed by the Scratch team and was considered not safe for everyone (nfe). It can't appear in search results, on the explore page and on the front page. + + notreviewed: The project hasn't been reviewed yet. + + no_remixes: Unable to fetch the project's moderation status. + """ + if self._moderation_status and (not reload): + return self._moderation_status + try: + return requests.get(f"https://jeffalo.net/api/nfe/?project={self.id}").json()["status"] + except Exception as exc: + raise exceptions.FetchError from exc + + def visibility(self): + """ + Returns info about the project's visibility. Requires authentication. + """ + session = self._assert_auth() + return requests.get( + f"https://api.scratch.mit.edu/users/{session.username}/projects/{self.id}/visibility", + headers=self._headers, + cookies=self._cookies, + ).json() + + +def get_project(project_id) -> Project: + """ + Gets a project without logging in. + + Args: + project_id (int): Project id of the requested project + + Returns: + scratchattach.project.Project: An object representing the requested project. + + Warning: + Any methods that require authentication (like project.love) will not work on the returned object. + + If you want to use these methods, get the project with :meth:`scratchattach.session.Session.connect_project` instead. + """ + warnings.warn( + "For methods that require authentication, use session.connect_project instead of get_project.\nIf you want to remove this warning, use `warnings.filterwarnings('ignore', category=scratchattach.ProjectAuthenticationWarning)`.\nTo ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use `warnings.filterwarnings('ignore', category=scratchattach.GetAuthenticationWarning)`.", + exceptions.ProjectAuthenticationWarning, + ) + return commons._get_object("id", project_id, Project, exceptions.ProjectNotFound) + + +def search_projects(*, query="", mode="trending", language="en", limit=40, offset=0): + """ + Uses the Scratch search to search projects. + + Keyword arguments: + query (str): The query that will be searched. + mode (str): Has to be one of these values: "trending", "popular" or "recent". Defaults to "trending". + language (str): A language abbreviation, defaults to "en". (Depending on the language used on the Scratch website, Scratch displays you different results.) + limit (int): Max. amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: List that contains the search results. + """ + if not query: + raise ValueError("The query can't be empty for search") + response = commons.api_iterative( + "https://api.scratch.mit.edu/search/projects", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}&q={query}", + ) + return commons.parse_object_list(response, Project) + + +def explore_projects(*, query="*", mode="trending", language="en", limit=40, offset=0): + """ + Gets projects from the explore page. + + Keyword arguments: + query (str): Specifies the tag of the explore page. To get the projects from the "All" tag, set this argument to "*". + mode (str): Has to be one of these values: "trending", "popular" or "recent". Defaults to "trending". + language (str): A language abbreviation, defaults to "en". (Depending on the language used on the Scratch website, Scratch displays you different explore pages.) + limit (int): Max. amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: List that contains the explore page projects. + """ + if not query: + raise ValueError("The query can't be empty for search") + response = commons.api_iterative( + "https://api.scratch.mit.edu/explore/projects", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}&q={query}", + ) + return commons.parse_object_list(response, Project) diff --git a/scratchattach/async_api/site/session.py b/scratchattach/async_api/site/session.py new file mode 100644 index 00000000..92998209 --- /dev/null +++ b/scratchattach/async_api/site/session.py @@ -0,0 +1,1409 @@ +"""Session class and login function""" + +from __future__ import annotations +from types import TracebackType +import base64 +import bs4 +import datetime +import hashlib +import json +import pathlib +import random +import re +import time +import warnings +import zlib +from dataclasses import dataclass, field +from typing import Literal, Optional, TypeVar, TYPE_CHECKING, overload, Any, Union, cast, Self +from contextlib import contextmanager +from threading import local +from scratchattach import editor + +Type = type +if TYPE_CHECKING: + from _typeshed import FileDescriptorOrPath, SupportsRead + from scratchattach.cloud._base import BaseCloud + + T = TypeVar("T", bound=BaseCloud) +else: + T = TypeVar("T") +from bs4 import BeautifulSoup, Tag +from typing_extensions import deprecated +from . import activity, classroom, forum, studio, user, project, backpack_asset, alert +from . import typed_dicts +from ._base import BaseSiteComponent, api_iterative +from scratchattach.cloud import cloud, _base +from scratchattach.eventhandlers import message_events, filterbot +from scratchattach.other import other_apis +from scratchattach.utils import commons, exceptions +from scratchattach.utils.commons import headers, empty_project_json, webscrape_count, get_class_sort_mode +from scratchattach._shared import http as shared_http +from ..primitives import http +from .browser_cookies import Browser, ANY, cookies_from_browser + +ratelimit_cache: dict[str, list[float]] = {} + + +def enforce_ratelimit(__type: str, name: str, amount: int = 5, duration: int = 60) -> None: + cache = ratelimit_cache + cache.setdefault(__type, []) + uses = cache[__type] + while uses and uses[-1] < time.time() - duration: + uses.pop() + if len(uses) < amount: + uses.insert(0, time.time()) + return + raise exceptions.RateLimitedError( + f"Rate limit for {name} exceeded.\nThis rate limit is enforced by scratchattach, not by the Scratch API.\nFor security reasons, it cannot be turned off.\n\nDon't spam-create studios or similar, it WILL get you banned." + ) + + +C = TypeVar("C", bound=BaseSiteComponent) + + +class UnauthSession: + http_session: http._HTTPSession + + +@dataclass +class Session(BaseSiteComponent[typed_dicts.SessionDict]): + """ + Represents a Scratch log in / session. Stores authentication data (session id and xtoken). + + Attributes: + id: The session id associated with the login + username: The username associated with the login + xtoken: The xtoken associated with the login + email: The email address associated with the logged in account + new_scratcher: True if the associated account is a new Scratcher + mute_status: Information about commenting restrictions of the associated account + banned: Returns True if the associated account is banned + """ + + http_session: http._HTTPSession = field(repr=False, kw_only=True) + username: str = field(repr=False, default="") + _user: Optional[user.User] = field(repr=False, default=None) + id: str = field(repr=False, default="") + session_string: Optional[str] = field(repr=False, default=None) + xtoken: Optional[str] = field(repr=False, default=None) + email: Optional[str] = field(repr=False, default=None) + new_scratcher: bool = field(repr=False, default=False) + mute_status: Any = field(repr=False, default=None) + banned: bool = field(repr=False, default=False) + time_created: datetime.datetime = field(repr=False, default=datetime.datetime.fromtimestamp(0.0)) + language: str = field(repr=False, default="en") + has_outstanding_email_confirmation: bool = field(repr=False, default=False) + is_teacher: bool = field(repr=False, default=False) + is_teacher_invitee: bool = field(repr=False, default=False) + ocular_token: Optional[str] = field(repr=False, default=None) + _session: Session | UnauthSession = field(kw_only=True, init=False) + + def __str__(self) -> str: + return f"-L {self.username}" + + def __rich__(self): + from rich.panel import Panel + from rich.table import Table + from rich import box + from rich.markup import escape + + ret = Table( + f"[link={self.connect_linked_user().url}]{escape(self.username)}[/]", f"Created: {self.time_created}", expand=True + ) + ret.add_row("Email", escape(str(self.email))) + ret.add_row("Language", escape(str(self.language))) + ret.add_row("Mute status", escape(str(self.mute_status))) + ret.add_row("New scratcher?", str(self.new_scratcher)) + ret.add_row("Banned?", str(self.banned)) + ret.add_row("Has outstanding email confirmation?", str(self.has_outstanding_email_confirmation)) + ret.add_row("Is teacher invitee?", str(self.is_teacher_invitee)) + ret.add_row("Is teacher?", str(self.is_teacher)) + return ret + + @property + def _username(self) -> str: + return self.username + + def __post_init__(self): + self.update_function = shared_http.HTTPMethod.POST + self.update_api = "https://scratch.mit.edu/session" + self._headers = dict(headers) + try: + self.id = json.loads(self.id) + except json.JSONDecodeError: + pass + self._cookies = { + "scratchsessionsid": self.id, + "scratchcsrftoken": "a", + "scratchlanguage": "en", + "accept": "application/json", + "Content-Type": "application/json", + } + self._update_http_cookies_and_headers() + if self.id: + self._process_session_id() + self._session = self + + async def _aenter(self) -> Self: + await self.http_session.__aenter__() + return self + + async def _aexit( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: + await self.http_session.__aexit__(exc_type, exc_val, exc_tb) + + def _enter(self) -> None: + raise TypeError("Use async with instead") + + def _exit( + self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] + ) -> None: + pass + + def _update_from_data(self, data: typed_dicts.SessionDict): + self.xtoken = data["user"]["token"] + self._headers["X-Token"] = self.xtoken + self.has_outstanding_email_confirmation = data["flags"]["has_outstanding_email_confirmation"] + self.email = data["user"]["email"] + self.new_scratcher = data["permissions"]["new_scratcher"] + self.is_teacher = data["permissions"]["educator"] + self.is_teacher_invitee = data["permissions"]["educator_invitee"] + self.mute_status = data["permissions"]["mute_status"] + self.username = data["user"]["username"] + self.banned = data["user"]["banned"] + if self.banned: + warnings.warn( + f"Warning: The account {self.username} you logged in to is BANNED. Some features may not work properly." + ) + if self.has_outstanding_email_confirmation: + warnings.warn( + f"Warning: The account {self.username} you logged is not email confirmed. Some features may not work properly." + ) + return True + + def _process_session_id(self): + assert self.id + data, self.time_created = decode_session_id(self.id) + self.username = data["username"] + self.xtoken = data["token"] + self._headers["X-Token"] = self.xtoken + self.language = data.get("_language", "en") + + def _assert_ocular_auth(self) -> str: + if not self.ocular_token: + raise ValueError( + f"No ocular token supplied for {self}! You can add one by using Session.set_ocular_token(YOUR_TOKEN)." + ) + return self.ocular_token + + def _update_http_cookies_and_headers(self): + self.http_session.clear_cookies() + self.http_session.update_cookies(self._cookies) + self.http_session.clear_headers() + self.http_session.update_headers(self._headers) + + def set_ocular_token(self, token: str): + self.ocular_token = token + + def connect_linked_user(self) -> user.User: + """ + Gets the user associated with the login / session. + + Warning: + The returned User object is cached. To ensure its attribute are up to date, you need to run .update() on it. + + Returns: + scratchattach.user.User: Object representing the user associated with the session. + """ + cached = hasattr(self, "_user") + if cached: + cached = self._user is not None + if not cached: + self._user = self.connect_user(self._username) + assert self._user is not None + return self._user + + def get_linked_user(self) -> user.User: + return self.connect_linked_user() + + async def set_country(self, country: str = "Antarctica"): + """ + Sets the profile country of the session's associated user + + Arguments: + country (str): The country to relocate to + """ + async with self.http_session.post( + "https://scratch.mit.edu/accounts/settings/", shared_http.options().data({"country": country}).value + ): + pass + + async def resend_email(self, password: str): + """ + Sends a request to resend a confirmation email for this session's account + + Keyword arguments: + password (str): Password associated with the session (not stored) + """ + async with self.http_session.post( + "https://scratch.mit.edu/accounts/email_change/", + shared_http.options().data({"email_address": await self.get_new_email_address(), "password": password}).value, + ): + pass + + async def get_new_email_address(self) -> str: + """ + Gets the (unconfirmed) email address that this session has requested to transfer to, if any, + otherwise the current address. + + Returns: + str: The email that this session wants to switch to + """ + async with self.http_session.get("https://scratch.mit.edu/accounts/email_change/") as response: + soup = BeautifulSoup(await response.text(), "html.parser") + email = None + for label_span in soup.find_all("span", {"class": "label"}): + if not isinstance(label_span, Tag): + continue + if not isinstance(label_span.parent, Tag): + continue + if label_span.contents[0] == "New Email Address": + return label_span.parent.contents[-1].text.strip("\n ") + elif label_span.contents[0] == "Current Email Address": + email = label_span.parent.contents[-1].text.strip("\n ") + if email is None: + for label_span in soup.select("form#email-change span.current-email"): + email = label_span.text + assert email is not None + return email + + async def logout(self): + """ + Sends a logout request to scratch. (Might not do anything, might log out this account on other ips/sessions.) + """ + async with self.http_session.post("https://scratch.mit.edu/accounts/logout/"): + pass + + async def set_featured_data( + self, project_id: Optional[int] | Literal[""], project_label: Optional[int] | Literal[""] = None + ): + """ + Sends a request to change your featured project area. + + Positional arguments: + project_id: None -> don't change; empty string -> set to latest project (this is what most accounts have); int -> set the featured project to the one with the corresponding ID. If you do not own that project, an error is raised. + project_lavel: None -> don't change; empty string -> "Featured project"; 0 -> "Featured Tutorial"; 1 -> "Work in progress"; 2 -> "Remix this!"; 3 -> "My favorite things"; 4 -> "Why I scratch" + + Returns: + list: List that contains all messages as Activity objects. + + """ + payload: dict[str, int | str] = {} + if project_label is not None: + payload["featured_project_label"] = str(project_label) + if project_id is not None: + payload["featured_project"] = project_id + async with self.http_session.put( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", shared_http.options().json(payload).value + ) as response: + data = await response.json() + if errors := data.get("errors"): + raise Exception(f"Backend responded with error: {(errors[0] if len(errors) == 1 else errors)}") + return data + + @property + def ocular_headers(self) -> dict[str, str]: + return { + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36", + "referer": "https://ocular.jeffalo.net/", + "authorization": self._assert_ocular_auth(), + } + + async def get_ocular_status(self) -> typed_dicts.OcularUserDict: + self._assert_ocular_auth() + async with self.http_session.get( + "https://my-ocular.jeffalo.net/auth/me", + shared_http.options().disregard_default_headers().disregard_default_cookies().headers(self.ocular_headers).value, + ) as response: + return cast(typed_dicts.OcularUserDict, await response.json()) + + async def set_ocular_status(self, status: Optional[str] = None, color: Optional[str] = None) -> None: + self._assert_ocular_auth() + old = await self.get_ocular_status() + payload = {"color": color or old["color"], "status": status or old["status"]} + async with self.http_session.put( + f"https://my-ocular.jeffalo.net/api/user/{old['name']}", + shared_http.options() + .disregard_default_headers() + .disregard_default_cookies() + .headers(self.ocular_headers) + .json(payload) + .value, + ) as response: + assert response.json() == {"ok": "user updated"}, ( + f"Error occured on setting ocular status. auth/me response: {old}" + ) + + async def messages(self, *, limit: int = 40, offset: int = 0, date_limit=None, filter_by=None) -> list[activity.Activity]: + """ + Returns the messages. + + Keyword arguments: + limit, offset, date_limit + filter_by (str or None): Can either be None (no filter), "comments", "projects", "studios" or "forums" + + Returns: + list: List that contains all messages as Activity objects. + """ + add_params = "" + if date_limit is not None: + add_params += f"&dateLimit={date_limit}" + if filter_by is not None: + add_params += f"&filter={filter_by}" + data: list[Any] = await api_iterative( + self, + f"https://api.scratch.mit.edu/users/{self._username}/messages", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + add_params=add_params, + ) + return activity.Activity.parse_object_list(data, self) + + def admin_messages(self, *, limit=40, offset=0) -> list[dict]: + """ + Returns your messages sent by the Scratch team (alerts). + """ + return commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/messages/admin", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + ) + + async def classroom_alerts( + self, _classroom: Optional[classroom.Classroom | int] = None, mode: str = "Last created", page: Optional[int] = None + ): + """ + Load and parse admin alerts, optionally for a specific class, using https://scratch.mit.edu/site-api/classrooms/alerts/ + + Returns: + list[alert.EducatorAlert]: A list of parsed EducatorAlert objects + """ + if isinstance(_classroom, classroom.Classroom): + _classroom = _classroom.id + if _classroom is None: + _classroom_str = "" + else: + _classroom_str = f"{_classroom}/" + ascsort, descsort = get_class_sort_mode(mode) + async with self.http_session.get( + f"https://scratch.mit.edu/site-api/classrooms/alerts/{_classroom_str}", + shared_http.options().params({"page": page, "ascsort": ascsort, "descsort": descsort}).value, + ) as response: + data = await response.json() + alerts = [alert.EducatorAlert.from_json(alert_data, self) for alert_data in data] + return alerts + + def clear_messages(self): + """ + Clears all messages. + """ + return requests.post( + "https://scratch.mit.edu/site-api/messages/messages-clear/", + headers=self._headers, + cookies=self._cookies, + timeout=10, + ).text + + def message_count(self) -> int: + """ + Returns the message count. + + Returns: + int: message count + """ + return json.loads( + requests.get( + f"https://scratch.mit.edu/messages/ajax/get-message-count/", + headers=self._headers, + cookies=self._cookies, + timeout=10, + ).text + )["msg_count"] + + def feed(self, *, limit=20, offset=0, date_limit=None) -> list[activity.Activity]: + """ + Returns the "What's happening" section (frontpage). + + Returns: + list: List that contains all "What's happening" entries as Activity objects + """ + add_params = "" + if date_limit is not None: + add_params = f"&dateLimit={date_limit}" + data = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/following/users/activity", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + add_params=add_params, + ) + return commons.parse_object_list(data, activity.Activity, self) + + def get_feed(self, *, limit=20, offset=0, date_limit=None): + return self.feed(limit=limit, offset=offset, date_limit=date_limit) + + def loved_by_followed_users(self, *, limit=40, offset=0) -> list[project.Project]: + """ + Returns the "Projects loved by Scratchers I'm following" section (frontpage). + + Returns: + list: List that contains all "Projects loved by Scratchers I'm following" + entries as Project objects + """ + data = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/following/users/loves", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + ) + return commons.parse_object_list(data, project.Project, self) + + def shared_by_followed_users(self, *, limit=40, offset=0) -> list[project.Project]: + """ + Returns the "Projects by Scratchers I'm following" section (frontpage). + This section is only visible to old accounts (until ~2018). + For newer users, this method will always return an empty list. + + Returns: + list: List that contains all "Projects by Scratchers I'm following" + entries as Project objects + """ + data = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/following/users/projects", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + ) + ret = commons.parse_object_list(data, project.Project, self) + if not ret: + warnings.warn( + f"`shared_by_followed_users` got empty list `[]`. Note that this method is not supported for accounts made after 2018." + ) + return ret + + def in_followed_studios(self, *, limit=40, offset=0) -> list["project.Project"]: + """ + Returns the "Projects in studios I'm following" section (frontpage). + This section is only visible to old accounts (until ~2018) + For newer users, this method will always return an empty list. + + Returns: + list: List that contains all "Projects in studios I'm following" section" + entries as Project objects + """ + data = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/following/studios/projects", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + ) + ret = commons.parse_object_list(data, project.Project, self) + if not ret: + warnings.warn( + f"`in_followed_studios` got empty list `[]`. Note that this method is not supported for accounts made after 2018." + ) + return ret + + def connect_empty_project_pb(self) -> editor.Project: + pb = editor.Project.from_json(empty_project_json) + pb._session = self + return pb + + def connect_pb_from_dict(self, project_json: dict) -> editor.Project: + pb = editor.Project.from_json(project_json) + pb._session = self + return pb + + def connect_pb_from_file(self, path_to_file) -> editor.Project: + pb = editor.Project.from_sb3(path_to_file) + pb._session = self + return pb + + @staticmethod + def download_asset(asset_id_with_file_ext, *, filename: Optional[str] = None, fp=""): + if not (fp.endswith("/") or fp.endswith("\\")): + fp = fp + "/" + try: + if filename is None: + filename = str(asset_id_with_file_ext) + response = requests.get("https://assets.scratch.mit.edu/" + str(asset_id_with_file_ext), timeout=10) + open(f"{fp}{filename}", "wb").write(response.content) + except Exception: + raise exceptions.FetchError("Failed to download asset") + + def upload_asset(self, asset_content, *, asset_id=None, file_ext=None): + data = asset_content if isinstance(asset_content, bytes) else open(asset_content, "rb").read() + if isinstance(asset_content, str): + file_ext = pathlib.Path(asset_content).suffix + file_ext = file_ext.replace(".", "") + if asset_id is None: + asset_id = hashlib.md5(data).hexdigest() + requests.post( + f"https://assets.scratch.mit.edu/{asset_id}.{file_ext}", + headers=self._headers, + cookies=self._cookies, + data=data, + timeout=10, + ) + + def search_projects( + self, *, query: str = "", mode: str = "trending", language: str = "en", limit: int = 40, offset: int = 0 + ) -> list[project.Project]: + """ + Uses the Scratch search to search projects. + + Keyword arguments: + query (str): The query that will be searched. + mode (str): Has to be one of these values: "trending", "popular" or "recent". Defaults to "trending". + language (str): A language abbreviation, defaults to "en". + (Depending on the language used on the Scratch website, Scratch displays you different results.) + limit (int): Max. amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: List that contains the search results. + """ + query = f"&q={query}" if query else "" + response = commons.api_iterative( + f"https://api.scratch.mit.edu/search/projects", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}{query}", + ) + return commons.parse_object_list(response, project.Project, self) + + def explore_projects( + self, *, query: str = "*", mode: str = "trending", language: str = "en", limit: int = 40, offset: int = 0 + ) -> list[project.Project]: + """ + Gets projects from the explore page. + + Keyword arguments: + query (str): Specifies the tag of the explore page. + To get the projects from the "All" tag, set this argument to "*". + mode (str): Has to be one of these values: "trending", "popular" or "recent". + Defaults to "trending". + language (str): A language abbreviation, defaults to "en". + (Depending on the language used on the Scratch website, Scratch displays you different explore pages.) + limit (int): Max. amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: List that contains the explore page projects. + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/explore/projects", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}&q={query}", + ) + return commons.parse_object_list(response, project.Project, self) + + def search_studios( + self, *, query: str = "", mode: str = "trending", language: str = "en", limit: int = 40, offset: int = 0 + ) -> list[studio.Studio]: + query = f"&q={query}" if query else "" + response = commons.api_iterative( + f"https://api.scratch.mit.edu/search/studios", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}{query}", + ) + return commons.parse_object_list(response, studio.Studio, self) + + def explore_studios( + self, *, query: str = "", mode: str = "trending", language: str = "en", limit: int = 40, offset: int = 0 + ) -> list[studio.Studio]: + query = f"&q={query}" if query else "" + response = commons.api_iterative( + f"https://api.scratch.mit.edu/explore/studios", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}{query}", + ) + return commons.parse_object_list(response, studio.Studio, self) + + def create_project( + self, *, title: Optional[str] = None, project_json: dict = empty_project_json, parent_id=None + ) -> project.Project: + """ + Creates a project on the Scratch website. + + Warning: + Don't spam this method - it WILL get you banned from Scratch. + To prevent accidental spam, a rate limit (5 projects per minute) is implemented for this function. + """ + enforce_ratelimit("create_scratch_project", "creating Scratch projects") + if title is None: + title = f"Untitled-{random.randint(0, 1 << 16)}" + params = {"is_remix": "0" if parent_id is None else "1", "original_id": parent_id, "title": title} + response = requests.post( + "https://projects.scratch.mit.edu/", params=params, cookies=self._cookies, headers=self._headers, json=project_json + ).json() + return self.connect_project(response["content-name"]) + + def create_studio(self, *, title: Optional[str] = None, description: Optional[str] = None) -> studio.Studio: + """ + Create a studio on the scratch website + + Warning: + Don't spam this method - it WILL get you banned from Scratch. + To prevent accidental spam, a rate limit (5 studios per minute) is implemented for this function. + """ + enforce_ratelimit("create_scratch_studio", "creating Scratch studios") + if self.new_scratcher: + raise exceptions.Unauthorized(f"\nNew scratchers (like {self.username}) cannot create studios.") + response = requests.post("https://scratch.mit.edu/studios/create/", cookies=self._cookies, headers=self._headers) + studio_id = webscrape_count(response.json()["redirect"], "/studios/", "/") + new_studio = self.connect_studio(studio_id) + if title is not None: + new_studio.set_title(title) + if description is not None: + new_studio.set_description(description) + return new_studio + + def create_class(self, title: str, desc: str = "") -> classroom.Classroom: + """ + Create a class on the scratch website + + Warning: + Don't spam this method - it WILL get you banned from Scratch. + To prevent accidental spam, a rate limit (5 classes per minute) is implemented for this function. + """ + enforce_ratelimit("create_scratch_class", "creating Scratch classes") + if not self.is_teacher: + raise exceptions.Unauthorized(f"{self.username} is not a teacher; can't create class") + data = requests.post( + "https://scratch.mit.edu/classes/create_classroom/", + json={"title": title, "description": desc}, + headers=self._headers, + cookies=self._cookies, + ).json() + class_id = data[0]["id"] + return self.connect_classroom(class_id) + + def mystuff_counts(self) -> tuple[int, int, int]: + """ + Gets the number of shared projects, unshared projects, and studios as listed on the mystuff page, + and returns them in that order. + + Example usage: + shared, unshared, studios = sess.mystuff_counts() + print(f"You have {shared} shared projects, {unshared} unshared projects, and are in {studios} studios") + """ + with requests.no_error_handling(): + resp = requests.get("https://scratch.mit.edu/mystuff/", headers=self._headers, cookies=self._cookies) + soup = bs4.BeautifulSoup(resp.text, "html.parser") + shared_elem = soup.select_one("span[data-content='shared-count']") + unshared_elem = soup.select_one("span[data-content='unshared-count']") + gallery_elem = soup.select_one("span[data-content='gallery-count']") + assert shared_elem is not None + assert unshared_elem is not None + assert gallery_elem is not None + shared: str = shared_elem.text.strip() + unshared: str = unshared_elem.text.strip() + gallery: str = gallery_elem.text.strip() + return (int(shared), int(unshared), int(gallery)) + + def mystuff_projects( + self, filter_arg: str = "all", *, page: int = 1, sort_by: str = "", descending: bool = True + ) -> list[project.Project]: + """ + Gets the projects from the "My stuff" page. + + Args: + filter_arg (str): Possible values for this parameter are "all", "shared", "unshared" and "trashed" + + Keyword Arguments: + page (int): The page of the "My Stuff" projects that should be returned + sort_by (str): The key the projects should be sorted based on. Possible values for this parameter are "" (then the projects are sorted based on last modified), "view_count", love_count", "remixers_count" (then the projects are sorted based on remix count) and "title" (then the projects are sorted based on title) + descending (boolean): Determines if the element with the highest key value (the key is specified in the sort_by argument) should be returned first. Defaults to True. + + Returns: + list: A list with the projects from the "My Stuff" page, each project is represented by a Project object. + """ + if descending: + ascsort = "" + descsort = sort_by + else: + ascsort = sort_by + descsort = "" + try: + targets = requests.get( + f"https://scratch.mit.edu/site-api/projects/{filter_arg}/?page={page}&ascsort={ascsort}&descsort={descsort}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + projects = [] + for target in targets: + projects.append( + project.Project( + id=target["pk"], + _session=self, + author_name=self._username, + comments_allowed=None, + instructions=None, + notes=None, + created=target["fields"]["datetime_created"], + last_modified=target["fields"]["datetime_modified"], + share_date=target["fields"]["datetime_shared"], + thumbnail_url="https:" + target["fields"]["thumbnail_url"], + favorites=target["fields"]["favorite_count"], + loves=target["fields"]["love_count"], + remix_count=target["fields"]["remixers_count"], + views=target["fields"]["view_count"], + title=target["fields"]["title"], + comment_count=target["fields"]["commenters_count"], + ) + ) + return projects + except Exception: + raise exceptions.FetchError() + + def mystuff_studios( + self, filter_arg: str = "all", *, page: int = 1, sort_by: str = "", descending: bool = True + ) -> list[studio.Studio]: + if descending: + ascsort = "" + descsort = sort_by + else: + ascsort = sort_by + descsort = "" + try: + params: dict[str, Union[str, int]] = {"page": page, "ascsort": ascsort, "descsort": descsort} + targets = requests.get( + f"https://scratch.mit.edu/site-api/galleries/{filter_arg}/", + params=params, + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + studios = [] + for target in targets: + studios.append( + studio.Studio( + id=target["pk"], + _session=self, + title=target["fields"]["title"], + description=None, + host_id=target["fields"]["owner"]["pk"], + host_name=target["fields"]["owner"]["username"], + open_to_all=None, + comments_allowed=None, + image_url="https:" + target["fields"]["thumbnail_url"], + created=target["fields"]["datetime_created"], + modified=target["fields"]["datetime_modified"], + follower_count=None, + manager_count=None, + curator_count=target["fields"]["curators_count"], + project_count=target["fields"]["projecters_count"], + ) + ) + return studios + except Exception: + raise exceptions.FetchError() + + def mystuff_classes_counts(self) -> tuple[int, int]: + """ + Returns the number of open and ended classes owned by a teacher session. + If this is not a teacher session, NotATeacherError is raised + """ + with requests.no_error_handling(): + resp = requests.get("https://scratch.mit.edu/educators/classes/", headers=self._headers, cookies=self._cookies) + if resp.status_code == 403: + raise exceptions.NotATeacherError("Response 403 when getting educators/classes") + soup = BeautifulSoup(resp.text, "html.parser") + sidebar = soup.find("div", {"id": "sidebar", "class": "tabs-index"}) + if not sidebar: + return (0, 0) + count_elem = sidebar.find("span", {"data-content": "classroom-count"}) + ended_elem = sidebar.find("span", {"data-content": "closed-count"}) + if not count_elem or not ended_elem: + return (0, 0) + count = str(count_elem.text).strip() + ended_count = str(ended_elem.text).strip() + return (int(count), int(ended_count)) + + def mystuff_classes(self, mode: str = "Last created", page: Optional[int] = None) -> list[classroom.Classroom]: + if not self.is_teacher: + self.update() + if not self.is_teacher: + raise exceptions.Unauthorized(f"{self.username} is not a teacher; can't have classes") + ascsort, descsort = get_class_sort_mode(mode) + classes_data = requests.get( + "https://scratch.mit.edu/site-api/classrooms/all/", + params={"page": page, "ascsort": ascsort, "descsort": descsort}, + headers=self._headers, + cookies=self._cookies, + ).json() + classes = [] + for data in classes_data: + fields = data["fields"] + educator_pf = fields["educator_profile"] + classes.append( + classroom.Classroom( + id=data["pk"], + title=fields["title"], + classtoken=fields["token"], + datetime=datetime.datetime.fromisoformat(fields["datetime_created"]), + author=user.User(username=educator_pf["user"]["username"], id=educator_pf["user"]["pk"], _session=self), + _session=self, + ) + ) + return classes + + def mystuff_ended_classes(self, mode: str = "Last created", page: Optional[int] = None) -> list[classroom.Classroom]: + if not self.is_teacher: + raise exceptions.Unauthorized(f"{self.username} is not a teacher; can't have (deleted) classes") + ascsort, descsort = get_class_sort_mode(mode) + classes_data = requests.get( + "https://scratch.mit.edu/site-api/classrooms/closed/", + params={"page": page, "ascsort": ascsort, "descsort": descsort}, + headers=self._headers, + cookies=self._cookies, + ).json() + classes = [] + for data in classes_data: + fields = data["fields"] + educator_pf = fields["educator_profile"] + classes.append( + classroom.Classroom( + id=data["pk"], + title=fields["title"], + classtoken=fields["token"], + datetime=datetime.datetime.fromisoformat(fields["datetime_created"]), + author=user.User(username=educator_pf["user"]["username"], id=educator_pf["user"]["pk"], _session=self), + _session=self, + ) + ) + return classes + + def backpack(self, limit: int = 20, offset: int = 0) -> list[backpack_asset.BackpackAsset]: + """ + Lists the assets that are in the backpack of the user associated with the session. + + Returns: + list: List that contains the backpack items + """ + data = commons.api_iterative( + f"https://backpack.scratch.mit.edu/{self._username}", limit=limit, offset=offset, _headers=self._headers + ) + return commons.parse_object_list(data, backpack_asset.BackpackAsset, self) + + def delete_from_backpack(self, backpack_asset_id) -> backpack_asset.BackpackAsset: + """ + Deletes an asset from the backpack. + + Args: + backpack_asset_id: ID of the backpack asset that will be deleted + """ + return backpack_asset.BackpackAsset(id=backpack_asset_id, _session=self).delete() + + def become_scratcher_invite(self) -> dict: + """ + If you are a new Scratcher and have been invited for becoming a Scratcher, this API endpoint will provide + more info on the invite. + """ + return requests.get( + f"https://api.scratch.mit.edu/users/{self.username}/invites", headers=self._headers, cookies=self._cookies + ).json() + + @overload + def connect_cloud(self, project_id, *, cloud_class: type[T]) -> T: + """ + Connects to a cloud as logged-in user. + + Args: + project_id: + + Keyword arguments: CloudClass: The class that the returned object should be of. By default, this class is + scratchattach.cloud.ScratchCloud. + + Returns: Type[scratchattach.cloud._base.BaseCloud]: An object representing the cloud of a project. Can be of any + class inheriting from BaseCloud. + """ + + @overload + def connect_cloud(self, project_id) -> cloud.ScratchCloud: + """ + Connects to a cloud (by default Scratch's cloud) as logged-in user. + + Args: + project_id: + + Keyword arguments: CloudClass: The class that the returned object should be of. By default, this class is + scratchattach.cloud.ScratchCloud. + + Returns: Type[scratchattach.cloud._base.BaseCloud]: An object representing the cloud of a project. Can be of any + class inheriting from BaseCloud. + """ + + def connect_cloud(self, project_id, *, cloud_class: Optional[type[_base.BaseCloud]] = None) -> _base.BaseCloud: + cloud_class = cloud_class or cloud.ScratchCloud + return cloud_class(project_id=project_id, _session=self) + + def connect_scratch_cloud(self, project_id) -> cloud.ScratchCloud: + """ + Returns: + scratchattach.cloud.ScratchCloud: An object representing the Scratch cloud of a project. + """ + return cloud.ScratchCloud(project_id=project_id, _session=self) + + def connect_tw_cloud( + self, project_id, *, purpose="", contact="", cloud_host="wss://clouddata.turbowarp.org" + ) -> cloud.TwCloud: + """ + Returns: + scratchattach.cloud.TwCloud: An object representing the TurboWarp cloud of a project. + """ + return cloud.TwCloud(project_id=project_id, purpose=purpose, contact=contact, cloud_host=cloud_host, _session=self) + + def _make_linked_object( + self, identificator_name, identificator, __class: type[C], NotFoundException: type[Exception] + ) -> C: + """ + The Session class doesn't save the login in a ._session attribute, but IS the login ITSELF. + + Therefore, the _make_linked_object method has to be adjusted + to get it to work for in the Session class. + + Class must inherit from BaseSiteComponent + """ + return commons._get_object(identificator_name, identificator, __class, NotFoundException, self) + + def connect_user(self, username: str) -> user.User: + """ + Gets a user using this session, connects the session to the User object to allow authenticated actions + + Args: + username (str): Username of the requested user + + Returns: + scratchattach.user.User: An object that represents the requested user and allows you to perform actions on the user (like user.follow) + """ + return self._make_linked_object("username", username, user.User, exceptions.UserNotFound) + + @deprecated("Finding usernames by user ids has been fixed.") + def find_username_from_id(self, user_id: int) -> str: + """ + Warning: + Every time this functions is run, a comment on your profile is posted and deleted. Therefore you shouldn't run this too often. + + Returns: + str: The username that corresponds to the user id + """ + you = user.User(username=self.username, _session=self) + try: + comment = you.post_comment("scratchattach", commentee_id=int(user_id)) + except exceptions.CommentPostFailure: + raise exceptions.BadRequest( + "After posting a comment, you need to wait 10 seconds before you can connect users by id again." + ) + except exceptions.BadRequest: + raise exceptions.UserNotFound("Invalid user id") + except Exception as e: + raise e + you.delete_comment(comment_id=comment.id) + try: + username = comment.content.split('">@')[1] + username = username.split("")[0] + except IndexError: + raise exceptions.UserNotFound() + return username + + @deprecated("Finding usernames by user ids has been fixed.") + def connect_user_by_id(self, user_id: int) -> user.User: + """ + Gets a user using this session, connects the session to the User object to allow authenticated actions + + This method ... + 1) gets the username by posting a comment with the user_id as commentee_id. + 2) deletes the posted comment. + 3) fetches other information about the user using Scratch's api.scratch.mit.edu/users/username API. + + Warning: + Every time this functions is run, a comment on your profile is posted and deleted. Therefore, you shouldn't run this too often. + + Args: + user_id (int): User ID of the requested user + + Returns: + scratchattach.user.User: An object that represents the requested user and allows you to perform actions on the user (like user.follow) + """ + return self._make_linked_object("username", self.find_username_from_id(user_id), user.User, exceptions.UserNotFound) + + def connect_project(self, project_id) -> project.Project: + """ + Gets a project using this session, connects the session to the Project object to allow authenticated actions + sess + Args: + project_id (int): ID of the requested project + + Returns: + scratchattach.project.Project: An object that represents the requested project and allows you to perform actions on the project (like project.love) + """ + return self._make_linked_object("id", int(project_id), project.Project, exceptions.ProjectNotFound) + + def connect_studio(self, studio_id) -> studio.Studio: + """ + Gets a studio using this session, connects the session to the Studio object to allow authenticated actions + + Args: + studio_id (int): ID of the requested studio + + Returns: + scratchattach.studio.Studio: An object that represents the requested studio and allows you to perform actions on the studio (like studio.follow) + """ + return self._make_linked_object("id", int(studio_id), studio.Studio, exceptions.StudioNotFound) + + def connect_classroom(self, class_id) -> classroom.Classroom: + """ + Gets a class using this session. + + Args: + class_id (str): class id of the requested class + + Returns: + scratchattach.classroom.Classroom: An object representing the requested classroom + """ + return self._make_linked_object("id", int(class_id), classroom.Classroom, exceptions.ClassroomNotFound) + + def connect_classroom_from_token(self, class_token) -> classroom.Classroom: + """ + Gets a class using this session. + + Args: + class_token (str): class token of the requested class + + Returns: + scratchattach.classroom.Classroom: An object representing the requested classroom + """ + return self._make_linked_object("classtoken", int(class_token), classroom.Classroom, exceptions.ClassroomNotFound) + + def connect_topic(self, topic_id) -> forum.ForumTopic: + """ + Gets a forum topic using this session, connects the session to the ForumTopic object to allow authenticated actions + Data is up-to-date. Data received from Scratch's RSS feed XML API. + + Args: + topic_id (int): ID of the requested forum topic (can be found in the browser URL bar) + + Returns: + scratchattach.forum.ForumTopic: An object that represents the requested forum topic + """ + return self._make_linked_object("id", int(topic_id), forum.ForumTopic, exceptions.ForumContentNotFound) + + def connect_topic_list(self, category_id, *, page=1): + """ + Gets the topics from a forum category. Data web-scraped from Scratch's forums UI. + Data is up-to-date. + + Args: + category_id (str): ID of the forum category + + Keyword Arguments: + page (str): Page of the category topics that should be returned + + Returns: + list: A list containing the forum topics from the specified category + """ + try: + response = requests.get( + f"https://scratch.mit.edu/discuss/{category_id}/?page={page}", headers=self._headers, cookies=self._cookies + ) + soup = BeautifulSoup(response.content, "html.parser") + except Exception as e: + raise exceptions.FetchError(str(e)) + try: + category_name = soup.find("h4").find("span").get_text() + except Exception: + raise exceptions.BadRequest("Invalid category id") + try: + topics = soup.find_all("tr") + topics.pop(0) + return_topics = [] + for topic in topics: + title_link = topic.find("a") + title = title_link.text.strip() + topic_id = title_link["href"].split("/")[-2] + columns = topic.find_all("td") + columns = [column.text for column in columns] + if len(columns) == 1: + continue + last_updated = columns[3].split(" ")[0] + " " + columns[3].split(" ")[1] + return_topics.append( + forum.ForumTopic( + _session=self, + id=int(topic_id), + title=title, + category_name=category_name, + last_updated=last_updated, + reply_count=int(columns[1]), + view_count=int(columns[2]), + ) + ) + return return_topics + except Exception as e: + raise exceptions.ScrapeError(str(e)) + + def connect_featured(self) -> other_apis.FeaturedData: + """ + Request and return connected featured projects and studios from the front page. + """ + return other_apis.get_featured_data(self) + + def connect_message_events(self, *, update_interval=2) -> message_events.MessageEvents: + return message_events.MessageEvents(user.User(username=self.username, _session=self), update_interval=update_interval) + + def connect_filterbot(self, *, log_deletions=True) -> filterbot.Filterbot: + return filterbot.Filterbot(user.User(username=self.username, _session=self), log_deletions=log_deletions) + + def get_session_string(self) -> str: + assert self.session_string + return self.session_string + + def get_headers(self) -> dict[str, str]: + return self._headers + + def get_cookies(self) -> dict[str, str]: + return self._cookies + + +@dataclass +class PreparedSession: + """ + Session that needs to be activated in a context manager first. Do not instantiate this yourself. + """ + + args: Any = field(repr=False) + kwargs: Any = field(repr=False) + _session: Session = field(repr=False, init=False) + + def __enter__(self) -> None: + raise TypeError("Use async with instead") + + def __exit__( + self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] + ) -> None: + pass + + async def __aenter__(self) -> Session: + self._session = await Session(*self.args, **self.kwargs | {"http_session": http._HTTPSession()})._aenter() + return self._session + + async def __aexit__( + self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None + ) -> None: + await self._session._aexit(exc_type, exc_val, exc_tb) + + +def decode_session_id(session_id: str) -> tuple[dict[str, str], datetime.datetime]: + """ + Extract the JSON data from the main part of a session ID string + Session id is in the format: + :: + + p1 contains a base64 JSON string (if it starts with `.`, then it is zlib compressed) + p2 is a base 62 encoded timestamp + p3 might be a `synchronous signature` for the first 2 parts (might be useless for us) + + The dict has these attributes: + - username + - _auth_user_id + - testcookie + - _auth_user_backend + - token + - login-ip + - _language + - django_timezone + - _auth_user_hash + """ + p1, p2, _ = session_id.split(":") + p1_bytes = base64.urlsafe_b64decode(p1 + "==") + if p1.startswith('".') or p1.startswith("."): + p1_bytes = zlib.decompress(p1_bytes) + return (json.loads(p1_bytes), datetime.datetime.fromtimestamp(commons.b62_decode(p2))) + + +_global_http_session: http._HTTPSession | None = None + + +async def _get_global_http_session() -> http._HTTPSession: + global _global_http_session + if _global_http_session is None: + async with http._HTTPSession() as session: + _global_http_session = session + return _global_http_session + + +suppressed_login_warning = local() + + +@contextmanager +def suppress_login_warning(): + """ + Suppress the login warning. + """ + suppressed_login_warning.suppressed = getattr(suppressed_login_warning, "suppressed", 0) + try: + suppressed_login_warning.suppressed += 1 + yield + finally: + suppressed_login_warning.suppressed -= 1 + + +def issue_login_warning() -> None: + """ + Issue a login data warning. + """ + if getattr(suppressed_login_warning, "suppressed", 0): + return + warnings.warn( + "IMPORTANT: If you included login credentials directly in your code (e.g. session_id, session_string, ...), then make sure to EITHER instead load them from environment variables or files OR remember to remove them before you share your code with anyone else. If you want to remove this warning, use `warnings.filterwarnings('ignore', category=scratchattach.LoginDataWarning)`", + exceptions.LoginDataWarning, + ) + + +def login_by_id( + session_id: str, *, username: Optional[str] = None, password: Optional[str] = None, xtoken=None +) -> PreparedSession: + """ + Creates a session / log in to the Scratch website with the specified session id. + Structured similarly to Session._connect_object method. + + Args: + session_id (str) + + Keyword arguments: + username (str) + password (str) + xtoken (str) + + Returns: + scratchattach.session.Session: An object that represents the created login / session + """ + issue_login_warning() + if password is not None: + session_data = dict(id=session_id, username=username, password=password) + session_string = base64.b64encode(json.dumps(session_data).encode()).decode() + else: + session_string = None + _session = PreparedSession((), {"id": session_id, "username": username or "", "session_string": session_string}) + return _session + + +async def login(username, password, *, timeout: float | int = 10) -> PreparedSession: + """ + Creates a session / log in to the Scratch website with the specified username and password. + + This method ... + 1. creates a session id by posting a login request to Scratch's login API. (If this fails, scratchattach.exceptions.LoginFailure is raised) + 2. fetches the xtoken and other information by posting a request to scratch.mit.edu/session. (If this fails, a warning is displayed) + + Args: + username (str) + password (str) + + Keyword arguments: + timeout (int): Timeout for the request to Scratch's login API (in seconds). Defaults to 10. + + Returns: + scratchattach.session.Session: An object that represents the created login / session + """ + issue_login_warning() + http_session = await _get_global_http_session() + _headers = headers.copy() + _headers["Cookie"] = "scratchcsrftoken=a;scratchlanguage=en;" + async with http_session.post( + "https://scratch.mit.edu/login/", + shared_http.options().headers(_headers).timeout(timeout).json({"username": username, "password": password}).value, + ) as response: + try: + result = re.search('"(.*)"', response.headers["Set-Cookie"]) + assert result is not None + session_id = str(result.group()) + except Exception: + raise exceptions.LoginFailure( + "Either the provided authentication data is wrong or your network is banned from Scratch.\n\nIf you're using an online IDE (like replit.com) Scratch possibly banned its IP address. In this case, try logging in with your session id: https://github.com/TimMcCool/scratchattach/wiki#logging-in" + ) + with suppress_login_warning(): + return login_by_id(session_id, username=username, password=password) + + +async def login_by_session_string(session_string: str) -> PreparedSession: + """ + Login using a session string. + """ + issue_login_warning() + session_string = base64.b64decode(session_string).decode() + session_data = json.loads(session_string) + try: + assert session_data.get("id") + with suppress_login_warning(): + return login_by_id( + session_data["id"], username=session_data.get("username"), password=session_data.get("password") + ) + except Exception: + pass + try: + assert session_data.get("session_id") + with suppress_login_warning(): + return login_by_id( + session_data["session_id"], username=session_data.get("username"), password=session_data.get("password") + ) + except Exception: + pass + try: + assert session_data.get("username") and session_data.get("password") + with suppress_login_warning(): + return await login(username=session_data["username"], password=session_data["password"]) + except Exception: + pass + raise ValueError("Couldn't log in.") + + +async def login_by_io(file: SupportsRead[str]) -> PreparedSession: + """ + Login using a file object. + """ + with suppress_login_warning(): + return await login_by_session_string(file.read()) + + +async def login_by_file(file: FileDescriptorOrPath) -> PreparedSession: + """ + Login using a path to a file. + """ + with suppress_login_warning(), open(file, encoding="utf-8") as f: + return await login_by_io(f) + + +def login_from_browser(browser: Browser = ANY) -> PreparedSession: + """ + Login from a browser + """ + cookies = cookies_from_browser(browser) + if "scratchsessionsid" in cookies: + with suppress_login_warning(): + return login_by_id(cookies["scratchsessionsid"]) + raise ValueError("Not enough data to log in.") diff --git a/scratchattach/async_api/site/studio.py b/scratchattach/async_api/site/studio.py new file mode 100644 index 00000000..160dc0cb --- /dev/null +++ b/scratchattach/async_api/site/studio.py @@ -0,0 +1,638 @@ +"""Studio class""" + +from __future__ import annotations +import warnings +import json +import random +from dataclasses import dataclass, field +from typing_extensions import Optional +from . import user, comment, project, activity, session +from scratchattach.site.typed_dicts import StudioDict, StudioRoleDict +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions, commons +from scratchattach.utils.commons import api_iterative, headers +from scratchattach.utils.requests import requests + + +@dataclass +class Studio(BaseSiteComponent[StudioDict]): + """ + Represents a Scratch studio. + """ + + id: int + title: Optional[str] = None + description: Optional[str] = None + host_id: Optional[int] = None + "The user id of the studio host" + follower_count: Optional[int] = None + manager_count: Optional[int] = None + project_count: Optional[int] = None + image_url: Optional[str] = None + open_to_all: Optional[bool] = None + "Whether everyone is allowed to add projects" + comments_allowed: Optional[bool] = None + created: Optional[str] = None + modified: Optional[str] = None + _session: Optional[session.Session] = None + + def __post_init__(self): + self.update_function = requests.get + self.update_api = f"https://api.scratch.mit.edu/studios/{self.id}" + if self._session is None: + self._headers = headers + self._cookies = {} + else: + self._headers = self._session._headers + self._cookies = self._session._cookies + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def _update_from_data(self, studio: StudioDict): + self.id = int(studio["id"]) + self.title = studio["title"] + self.description = studio["description"] + self.host_id = studio["host"] + self.open_to_all = studio["open_to_all"] + self.comments_allowed = studio["comments_allowed"] + self.image_url = studio["image"] + self.created = studio["history"]["created"] + self.modified = studio["history"]["modified"] + stats = studio.get("stats", {}) + self.follower_count = stats.get("followers", self.follower_count) + self.manager_count = stats.get("managers", self.manager_count) + self.project_count = stats.get("projects", self.project_count) + return True + + def __str__(self): + ret = f"-S {self.id}" + if self.title: + ret += f" ({self.title})" + return ret + + def __rich__(self): + from rich.panel import Panel + from rich.table import Table + from rich import box + from rich.markup import escape + + url = f"[link={self.url}]{escape(self.title)}[/]" + ret = Table.grid(expand=True) + ret.add_column(ratio=1) + ret.add_column(ratio=3) + info = Table(box=box.SIMPLE) + info.add_column(url, overflow="fold") + info.add_column(f"#{self.id}", overflow="fold") + info.add_row("Host ID", str(self.host_id)) + info.add_row("Followers", str(self.follower_count)) + info.add_row("Projects", str(self.project_count)) + info.add_row("Managers", str(self.manager_count)) + info.add_row("Comments allowed", str(self.comments_allowed)) + info.add_row("Open", str(self.open_to_all)) + info.add_row("Created", self.created) + info.add_row("Modified", self.modified) + desc = Table(box=box.SIMPLE) + desc.add_row("Description", escape(self.description)) + ret.add_row(Panel(info, title=url), Panel(desc, title="Description")) + return ret + + @property + def url(self): + return f"https://scratch.mit.edu/studios/{self.id}" + + @property + def thumbnail(self) -> bytes: + with requests.no_error_handling(): + return requests.get(self.image_url).content + + def follow(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/users/bookmarkers/{self.id}/add/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + + def unfollow(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/users/bookmarkers/{self.id}/remove/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + + def comments(self, *, limit=40, offset=0) -> list[comment.Comment]: + """ + Returns the comments posted on the studio (except for replies. To get replies use :meth:`scratchattach.studio.Studio.get_comment_replies`). + + Keyword Arguments: + page: The page of the comments that should be returned. + limit: Max. amount of returned comments. + + Returns: + list: A list containing the requested comments as Comment objects. + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/studios/{self.id}/comments/", + limit=limit, + offset=offset, + add_params=f"&cachebust={random.randint(0, 9999)}", + ) + for i in response: + i["source"] = "studio" + i["source_id"] = self.id + return commons.parse_object_list(response, comment.Comment, self._session) + + def comment_replies(self, *, comment_id, limit=40, offset=0) -> list[comment.Comment]: + response = commons.api_iterative( + f"https://api.scratch.mit.edu/studios/{self.id}/comments/{comment_id}/replies", + limit=limit, + offset=offset, + add_params=f"&cachebust={random.randint(0, 9999)}", + ) + for x in response: + x["parent_id"] = comment_id + x["source"] = "studio" + x["source_id"] = self.id + return commons.parse_object_list(response, comment.Comment, self._session) + + def comment_by_id(self, comment_id): + r = requests.get(f"https://api.scratch.mit.edu/studios/{self.id}/comments/{comment_id}", timeout=10).json() + if r is None: + raise exceptions.CommentNotFound() + _comment = comment.Comment(id=r["id"], _session=self._session, source=comment.CommentSource.STUDIO, source_id=self.id) + _comment._update_from_data(r) + return _comment + + def post_comment(self, content, *, parent_id="", commentee_id=""): + """ + Posts a comment on the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Args: + content: Content of the comment that should be posted + + Keyword Arguments: + parent_id: ID of the comment you want to reply to. If you don't want to mention a user, don't put the argument. + commentee_id: ID of the user that will be mentioned in your comment and will receive a message about your comment. If you don't want to mention a user, don't put the argument. + + Returns: + scratchattach.comment.Comment: The posted comment as Comment object. + """ + self._assert_auth() + data = {"commentee_id": commentee_id, "content": str(content), "parent_id": parent_id} + headers = dict(self._json_headers) + headers["referer"] = "https://scratch.mit.edu/projects/" + str(self.id) + "/" + r = requests.post( + f"https://api.scratch.mit.edu/proxy/comments/studio/{self.id}/", + headers=headers, + cookies=self._cookies, + data=json.dumps(data), + timeout=10, + ).json() + if "id" not in r: + raise exceptions.CommentPostFailure(r) + _comment = comment.Comment(id=r["id"], _session=self._session, source=comment.CommentSource.STUDIO, source_id=self.id) + _comment._update_from_data(r) + return _comment + + def delete_comment(self, *, comment_id): + """ + Deletes a comment by ID. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + comment_id: The id of the comment that should be deleted + """ + self._assert_auth() + return requests.delete( + f"https://api.scratch.mit.edu/proxy/comments/studio/{self.id}/comment/{comment_id}/", + headers=self._headers, + cookies=self._cookies, + ).headers + + def report_comment(self, *, comment_id): + """ + Reports a comment by ID to the Scratch team. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + comment_id: The id of the comment that should be reported + """ + self._assert_auth() + return requests.delete( + f"https://api.scratch.mit.edu/proxy/comments/studio/{self.id}/comment/{comment_id}/report", + headers=self._headers, + cookies=self._cookies, + ) + + def set_thumbnail(self, *, file): + """ + Sets the studio thumbnail. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Keyword Arguments: + file: The path to the image file + + Returns: + str: Scratch cdn link to the set thumbnail + """ + self._assert_auth() + with open(file, "rb") as f: + thumbnail = f.read() + filename = file.replace("\\", "/") + if filename.endswith("/"): + filename = filename[:-1] + filename = filename.split("/").pop() + file_type = filename.split(".").pop() + payload1 = f'------WebKitFormBoundaryhKZwFjoxAyUTMlSh\r\nContent-Disposition: form-data; name="file"; filename="{filename}"\r\nContent-Type: image/{file_type}\r\n\r\n' + payload1 = payload1.encode("utf-8") + payload2 = b"\r\n------WebKitFormBoundaryhKZwFjoxAyUTMlSh--\r\n" + payload = b"".join([payload1, thumbnail, payload2]) + r = requests.post( + f"https://scratch.mit.edu/site-api/galleries/all/{self.id}/", + headers={ + "accept": "*/", + "content-type": "multipart/form-data; boundary=----WebKitFormBoundaryhKZwFjoxAyUTMlSh", + "Referer": "https://scratch.mit.edu/", + "x-csrftoken": "a", + "x-requested-with": "XMLHttpRequest", + }, + data=payload, + cookies=self._cookies, + timeout=10, + ).json() + if "errors" in r: + raise exceptions.BadRequest(", ".join(r["errors"])) + else: + return r["thumbnail_url"] + + def reply_comment(self, content, *, parent_id, commentee_id=""): + """ + Posts a reply to a comment on the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Args: + content: Content of the comment that should be posted + + Warning: + Only replies to top-level comments are shown on the Scratch website. Replies to replies are actually replies to the corresponding top-level comment in the API. + + Therefore, parent_id should be the comment id of a top level comment. + + Keyword Arguments: + parent_id: ID of the comment you want to reply to + commentee_id: ID of the user you are replying to + """ + self._assert_auth() + return self.post_comment(content, parent_id=parent_id, commentee_id=commentee_id) + + def projects(self, limit=40, offset=0) -> list[project.Project]: + """ + Gets the studio projects. + + Keyword arguments: + limit (int): Max amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: A list containing the studio projects as Project objects + """ + response = commons.api_iterative(f"https://api.scratch.mit.edu/studios/{self.id}/projects", limit=limit, offset=offset) + return commons.parse_object_list(response, project.Project, self._session) + + def curators(self, limit=40, offset=0) -> list[user.User]: + """ + Gets the studio curators. + + Keyword arguments: + limit (int): Max amount of returned curators. + offset (int): Offset of the first returned curator. + + Returns: + list: A list containing the studio curators as User objects + """ + response = commons.api_iterative(f"https://api.scratch.mit.edu/studios/{self.id}/curators", limit=limit, offset=offset) + return commons.parse_object_list(response, user.User, self._session, "username") + + def invite_curator(self, curator): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + try: + return requests.put( + f"https://scratch.mit.edu/site-api/users/curators-in/{self.id}/invite_curator/?usernames={curator}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + except Exception: + raise exceptions.Unauthorized + + def promote_curator(self, curator): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + try: + return requests.put( + f"https://scratch.mit.edu/site-api/users/curators-in/{self.id}/promote/?usernames={curator}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + except Exception: + raise exceptions.Unauthorized + + def remove_curator(self, curator): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + try: + return requests.put( + f"https://scratch.mit.edu/site-api/users/curators-in/{self.id}/remove/?usernames={curator}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + except Exception: + raise exceptions.Unauthorized + + def transfer_ownership(self, new_owner, *, password): + """ + Makes another Scratcher studio host. You need to specify your password to do this. + + Arguments: + new_owner (str): Username of new host + + Keyword arguments: + password (str): The password of your Scratch account + + Warning: + This action is irreversible! + """ + self._assert_auth() + try: + return requests.put( + f"https://api.scratch.mit.edu/studios/{self.id}/transfer/{new_owner}", + headers=self._headers, + cookies=self._cookies, + timeout=10, + json={"password": password}, + ).json() + except Exception: + raise exceptions.Unauthorized + + def leave(self): + """ + Removes yourself from the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + return self.remove_curator(self._session._username) + + def add_project(self, project_id): + """ + Adds a project to the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Args: + project_id: Project id of the project that should be added + """ + self._assert_auth() + return requests.post( + f"https://api.scratch.mit.edu/studios/{self.id}/project/{project_id}", headers=self._headers, timeout=10 + ).json() + + def remove_project(self, project_id): + """ + Removes a project from the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Args: + project_id: Project id of the project that should be removed + """ + self._assert_auth() + return requests.delete( + f"https://api.scratch.mit.edu/studios/{self.id}/project/{project_id}", headers=self._headers, timeout=10 + ).json() + + def managers(self, limit=40, offset=0): + """ + Gets the studio managers. + + Keyword arguments: + limit (int): Max amount of returned managers + offset (int): Offset of the first returned manager. + + Returns: + list: A list containing the studio managers as user objects + """ + response = commons.api_iterative(f"https://api.scratch.mit.edu/studios/{self.id}/managers", limit=limit, offset=offset) + return commons.parse_object_list(response, user.User, self._session, "username") + + def host(self) -> user.User: + """ + Gets the studio host. + + Returns: + scratchattach.user.User: An object representing the studio host. + """ + managers = self.managers(limit=1, offset=0) + try: + return managers[0] + except Exception: + return None + + def set_fields(self, fields_dict): + """ + Sets fields. Uses the scratch.mit.edu/site-api PUT API. + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/galleries/all/{self.id}/", + headers=headers, + cookies=self._cookies, + data=json.dumps(fields_dict), + timeout=10, + ) + + def set_description(self, new): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self.set_fields({"description": new + "\n"}) + + def set_title(self, new): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self.set_fields({"title": new}) + + def open_projects(self): + """ + Changes the studio settings so everyone (including non-curators) is able to add projects to the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/galleries/{self.id}/mark/open/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + + def close_projects(self): + """ + Changes the studio settings so only curators can add projects to the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/galleries/{self.id}/mark/closed/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + + def turn_off_commenting(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + if self.comments_allowed: + requests.post( + f"https://scratch.mit.edu/site-api/comments/gallery/{self.id}/toggle-comments/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + self.comments_allowed = not self.comments_allowed + + def turn_on_commenting(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + if not self.comments_allowed: + requests.post( + f"https://scratch.mit.edu/site-api/comments/gallery/{self.id}/toggle-comments/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + self.comments_allowed = not self.comments_allowed + + def toggle_commenting(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.post( + f"https://scratch.mit.edu/site-api/comments/gallery/{self.id}/toggle-comments/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + self.comments_allowed = not self.comments_allowed + + def activity(self, *, limit=40, offset=0, date_limit=None): + add_params = "" + if date_limit is not None: + add_params = f"&dateLimit={date_limit}" + response = commons.api_iterative( + f"https://api.scratch.mit.edu/studios/{self.id}/activity", limit=limit, offset=offset, add_params=add_params + ) + return commons.parse_object_list(response, activity.Activity, self._session) + + def accept_invite(self): + self._assert_auth() + return requests.put( + f"https://scratch.mit.edu/site-api/users/curators-in/{self.id}/add/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + + def your_role(self) -> StudioRoleDict: + """ + Returns a dict with information about your role in the studio (whether you're following, curating, managing it or are invited) + """ + self._assert_auth() + return requests.get( + f"https://api.scratch.mit.edu/studios/{self.id}/users/{self._session.username}", + headers=self._headers, + cookies=self._cookies, + timeout=10, + ).json() + + def get_exact_project_count(self) -> int: + """ + Get the exact project count of a studio using a binary-search-like strategy + """ + if self.project_count is not None and self.project_count < 100: + return self.project_count + maximum = 100 + minimum = 0 + while True: + if not self.projects(offset=maximum): + break + minimum = maximum + maximum *= 2 + while True: + middle = (minimum + maximum) // 2 + projects = self.projects(limit=40, offset=middle) + if not projects: + maximum = middle + elif len(projects) < 40: + break + else: + minimum = middle + return middle + len(projects) + + +def get_studio(studio_id) -> Studio: + """ + Gets a studio without logging in. + + Args: + studio_id (int): Studio id of the requested studio + + Returns: + scratchattach.studio.Studio: An object representing the requested studio + + Warning: + Any methods that authentication (like studio.follow) will not work on the returned object. + + If you want to use these, get the studio with :meth:`scratchattach.session.Session.connect_studio` instead. + """ + warnings.warn( + "Warning: For methods that require authentication, use session.connect_studio instead of get_studio.\nIf you want to remove this warning, use warnings.filterwarnings('ignore', category=scratchattach.StudioAuthenticationWarning).\nTo ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use `warnings.filterwarnings('ignore', category=scratchattach.GetAuthenticationWarning)`.", + exceptions.StudioAuthenticationWarning, + ) + return commons._get_object("id", studio_id, Studio, exceptions.StudioNotFound) + + +def search_studios(*, query="", mode="trending", language="en", limit=40, offset=0): + if not query: + raise ValueError("The query can't be empty for search") + response = commons.api_iterative( + f"https://api.scratch.mit.edu/search/studios", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}&q={query}", + ) + return commons.parse_object_list(response, Studio) + + +def explore_studios(*, query="", mode="trending", language="en", limit=40, offset=0): + if not query: + raise ValueError("The query can't be empty for explore") + response = commons.api_iterative( + f"https://api.scratch.mit.edu/explore/studios", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}&q={query}", + ) + return commons.parse_object_list(response, Studio) diff --git a/scratchattach/async_api/site/typed_dicts.py b/scratchattach/async_api/site/typed_dicts.py new file mode 100644 index 00000000..e56433b4 --- /dev/null +++ b/scratchattach/async_api/site/typed_dicts.py @@ -0,0 +1,219 @@ +from __future__ import annotations +from scratchattach.cloud import _base +from typing import TypedDict, Required, NotRequired + + +class SessionUserDict(TypedDict): + id: int + banned: bool + should_vpn: bool + username: str + token: str + thumbnailUrl: str + dateJoined: str + email: str + birthYear: int + birthMonth: int + gender: str + + +class SessionOffenseDict(TypedDict): + expiresAt: float + messageType: str + createdAt: float + + +class EmptySessionOffensesDict(TypedDict): + pass + + +class SessionOffensesDict(TypedDict): + offenses: list[SessionOffenseDict] + showWarning: bool + muteExpiresAt: float + currentMessageType: str + + +class SessionPermissionsDict(TypedDict): + admin: bool + scratcher: bool + new_scratcher: bool + invited_scratcher: bool + social: bool + educator: bool + educator_invitee: bool + student: bool + mute_status: EmptySessionOffensesDict | SessionOffensesDict + + +class SessionFlagsDict(TypedDict): + must_reset_password: bool + must_complete_registration: bool + has_outstanding_email_confirmation: bool + show_welcome: bool + confirm_email_banner: bool + unsupported_browser_banner: bool + with_parent_email: bool + project_comments_enabled: bool + gallery_comments_enabled: bool + userprofile_comments_enabled: bool + everything_is_totally_normal: bool + + +class SessionDict(TypedDict): + user: SessionUserDict + permissions: SessionPermissionsDict + flags: SessionFlagsDict + + +class OcularUserMetaDict(TypedDict): + updated: str + updatedBy: str + + +class OcularUserDict(TypedDict): + _id: str + name: str + status: str + color: str + meta: OcularUserMetaDict + + +class UserHistoryDict(TypedDict): + joined: str + + +UserProfileImagesDict = TypedDict( + "UserProfileImagesDict", {"90x90": str, "60x60": str, "55x55": str, "50x50": str, "32x32": str} +) + + +class UserProfileDict(TypedDict): + id: int + status: str + bio: str + country: str + images: UserProfileImagesDict + membership_label: NotRequired[int] + membership_avatar_badge: NotRequired[int] + + +class UserDict(TypedDict): + id: NotRequired[int] + username: NotRequired[str] + scratchteam: NotRequired[bool] + history: NotRequired[UserHistoryDict] + profile: NotRequired[UserProfileDict] + + +class CloudLogActivityDict(TypedDict): + user: str + verb: str + name: str + variable_name: NotRequired[str] + value: str | float | int + timestamp: int + cloud: _base.AnyCloud + + +class CloudActivityDict(TypedDict): + method: str + name: str + variable_name: NotRequired[str] + value: str | float | int + project_id: int + cloud: _base.AnyCloud + + +class ClassroomDict(TypedDict): + id: int + title: str + description: str + status: str + date_start: NotRequired[str] + date_end: NotRequired[str | None] + images: NotRequired[dict[str, str]] + educator: UserDict + is_closed: NotRequired[bool] + + +class StudioHistoryDict(TypedDict): + created: str + modified: str + + +class StudioStatsDict(TypedDict): + followers: int + managers: int + projects: int + + +class StudioDict(TypedDict): + id: int + title: str + description: str + host: int + open_to_all: bool + comments_allowed: bool + image: str + history: StudioHistoryDict + stats: NotRequired[StudioStatsDict] + + +class StudioRoleDict(TypedDict): + manager: bool + curator: bool + invited: bool + following: bool + + +ProjectImagesDict = TypedDict( + "ProjectImagesDict", {"282x218": str, "216x163": str, "200x200": str, "144x108": str, "135x102": str, "100x80": str} +) + + +class ProjectHistoryDict(TypedDict): + created: str + modified: str + shared: str + + +class ProjectStatsDict(TypedDict): + views: int + loves: int + favorites: int + remixes: int + + +class ProjectRemixDict(TypedDict): + parent: int | None + root: int | None + + +class ProjectDict(TypedDict): + id: int + title: str + description: str + instructions: str + visibility: str + public: bool + comments_allowed: bool + is_published: bool + author: UserDict + image: str + images: ProjectImagesDict + history: ProjectHistoryDict + stats: ProjectStatsDict + remix: ProjectRemixDict + project_token: str + + +class PlaceholderProjectDataMetadataDict(TypedDict): + title: str + description: str + + +class PlaceholderProjectDataDict(TypedDict): + metadata: PlaceholderProjectDataMetadataDict + md5extsToSha256: dict[str, str] + adminOwnershipToken: str | None diff --git a/scratchattach/async_api/site/user.py b/scratchattach/async_api/site/user.py new file mode 100644 index 00000000..c21085d2 --- /dev/null +++ b/scratchattach/async_api/site/user.py @@ -0,0 +1,1018 @@ +"""User class""" + +from __future__ import annotations +import json +import random +import re +import string +import warnings +from typing import Union, cast, Optional, TypedDict +from dataclasses import dataclass, field +from datetime import datetime, timezone +from enum import Enum +from typing_extensions import deprecated +from bs4 import BeautifulSoup, Tag +from ._base import BaseSiteComponent +from scratchattach.eventhandlers import message_events +from scratchattach.utils import commons +from scratchattach.utils import exceptions +from scratchattach.utils.commons import headers +from scratchattach.utils.requests import requests +from . import project +from . import studio +from . import forum +from . import comment +from . import activity +from . import classroom +from . import typed_dicts +from . import session + + +class Rank(Enum): + """ + Possible ranks in scratch + """ + + NEW_SCRATCHER = 0 + SCRATCHER = 1 + SCRATCH_TEAM = 2 + + +class _OcularStatusMeta(TypedDict): + updated: str + updatedBy: str + + +class _OcularStatus(TypedDict): + _id: str + name: str + status: str + color: str + meta: _OcularStatusMeta + + +class Verificator: + def __init__(self, user: User, project_id: int): + self.project = user._make_linked_object("id", project_id, project.Project, exceptions.ProjectNotFound) + self.projecturl = self.project.url + self.code = "".join(random.choices(string.ascii_letters + string.digits, k=8)) + self.username = user.username + + def check(self) -> bool: + return bool( + list( + filter( + lambda x: ( + x.author_name == self.username + and (x.content == self.code or x.content.startswith(self.code) or x.content.endswith(self.code)) + ), + self.project.comments(), + ) + ) + ) + + +@dataclass +class User(BaseSiteComponent[typed_dicts.UserDict]): + """ + Represents a Scratch user. + + Attributes: + + :.join_date: + + :.about_me: + + :.wiwo: Returns the user's 'What I'm working on' section + + :.country: Returns the country from the user profile + + :.icon_url: Returns the link to the user's pfp (90x90) + + :.id: Returns the id of the user + + :.scratchteam: Retuns True if the user is in the Scratch team + + :.update(): Updates the attributes + """ + + username: str = field(kw_only=True, default="") + join_date: str = field(kw_only=True, default="") + about_me: str = field(kw_only=True, default="") + wiwo: str = field(kw_only=True, default="") + country: str = field(kw_only=True, default="") + icon_url: str = field(kw_only=True, default="") + id: int = field(kw_only=True, default=0) + scratchteam: bool = field(kw_only=True, repr=False, default=False) + is_member: bool = field(kw_only=True, repr=False, default=False) + has_ears: bool = field(kw_only=True, repr=False, default=False) + _classroom: tuple[bool, Optional[classroom.Classroom]] = field(init=False, default=(False, None)) + _headers: dict[str, str] = field(init=False, default_factory=headers.copy) + _cookies: dict[str, str] = field(init=False, default_factory=dict) + _json_headers: dict[str, str] = field(init=False, default_factory=dict) + _session: Optional[session.Session] = field(kw_only=True, default=None) + + def __str__(self): + return f"-U {self.username}" + + @property + def status(self) -> str: + return self.wiwo + + @property + def bio(self) -> str: + return self.about_me + + @property + def icon(self) -> bytes: + with requests.no_error_handling(): + return requests.get(self.icon_url).content + + @property + def name(self) -> str: + return self.username + + def __post_init__(self): + self.update_function = requests.get + self.update_api = f"https://api.scratch.mit.edu/users/{self.username}" + if self._session is not None: + self._headers = self._session.get_headers() + self._cookies = self._session.get_cookies() + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def _update_from_data(self, data: Union[dict, typed_dicts.UserDict]): + data = cast(typed_dicts.UserDict, data) + self.id = data.get("id", self.id) + self.username = data.get("username", self.username) + self.scratchteam = data.get("scratchteam", self.scratchteam) + if history := data.get("history"): + self.join_date = history["joined"] + if profile := data.get("profile"): + self.about_me = profile["bio"] + self.wiwo = profile["status"] + self.country = profile["country"] + self.icon_url = profile["images"]["90x90"] + self.is_member = bool(profile.get("membership_label", False)) + self.has_ears = bool(profile.get("membership_avatar_badge", False)) + return True + + def _assert_permission(self): + self._assert_auth() + if self._session.username != self.username: + raise exceptions.Unauthorized("You need to be authenticated as the profile owner to do this.") + + @property + def url(self): + return f"https://scratch.mit.edu/users/{self.username}" + + def __rich__(self): + from rich.panel import Panel + from rich.table import Table + from rich import box + from rich.markup import escape + + featured_data = self.featured_data() or {} + ocular_data = {} + ocular = "No ocular status" + if status := ocular_data.get("status"): + color_str = "" + color_data = ocular_data.get("color") + if color_data is not None: + color_str = f"[{color_data}] ⬤ [/]" + ocular = f"[i]{escape(status)}[/]{color_str}" + _classroom = self.classroom + url = f"[link={self.url}]{escape(self.username)}[/]" + info = Table(box=box.SIMPLE) + info.add_column(url, overflow="fold") + info.add_column(f"#{self.id}", overflow="fold") + info.add_row("Joined", escape(self.join_date)) + info.add_row("Country", escape(self.country)) + info.add_row("Messages", str(self.message_count())) + info.add_row("Class", str(_classroom.title if _classroom is not None else "None")) + desc = Table("Profile", ocular, box=box.SIMPLE) + desc.add_row("About me", escape(self.about_me)) + desc.add_row("Wiwo", escape(self.wiwo)) + desc.add_row(escape(featured_data.get("label", "Featured Project")), escape(str(self.connect_featured_project()))) + ret = Table.grid(expand=True) + ret.add_column(ratio=1) + ret.add_column(ratio=3) + ret.add_row(Panel(info, title=url), Panel(desc, title="Description")) + return ret + + def connect_featured_project(self) -> Optional[project.Project]: + data = self.featured_data() or {} + if pid := data.get("id"): + return self._session.connect_project(int(pid)) + if projs := self.projects(limit=1): + return projs[0] + return None + + @property + def classroom(self) -> classroom.Classroom | None: + """ + Get a user's associated classroom, and return it as a `scratchattach.classroom.Classroom` object. + If there is no associated classroom, returns `None` + """ + if not self._classroom[0]: + with requests.no_error_handling(): + resp = requests.get(f"https://scratch.mit.edu/users/{self.username}/") + soup = BeautifulSoup(resp.text, "html.parser") + details = soup.find("p", {"class": "profile-details"}) + if details is None: + return None + assert isinstance(details, Tag) + class_name, class_id, is_closed = (None, None, False) + for a in details.find_all("a"): + if not isinstance(a, Tag): + continue + href = str(a.get("href")) + if re.match("/classes/\\d*/", href): + class_name = a.text.strip()[len("Student of: ") :] + is_closed = bool(re.search("\\n *\\(ended\\)", class_name)) + if is_closed: + class_name = re.sub("\\n *\\(ended\\)", "", class_name).strip() + class_id = int(href.split("/")[2]) + break + if class_name: + self._classroom = ( + True, + classroom.Classroom(_session=self._session, id=class_id or 0, title=class_name, is_closed=is_closed), + ) + else: + self._classroom = (True, None) + return self._classroom[1] + + def does_exist(self) -> Optional[bool]: + """ + Returns: + boolean : True if the user exists, False if the user is deleted, None if an error occured + """ + with requests.no_error_handling(): + status_code = requests.get(f"https://scratch.mit.edu/users/{self.username}/").status_code + if status_code == 200: + return True + elif status_code == 404: + return False + return None + + def is_new_scratcher(self): + """ + Returns: + boolean : True if the user has the New Scratcher status, else False + """ + try: + with requests.no_error_handling(): + res = requests.get(f"https://scratch.mit.edu/users/{self.username}/").text + group = res[res.rindex('') :][:70] + return "new scratcher" in group.lower() + except Exception as e: + warnings.warn(f"Caught exception e={e!r}") + return None + + def message_count(self): + return json.loads( + requests.get( + f"https://api.scratch.mit.edu/users/{self.username}/messages/count/?cachebust={random.randint(0, 10000)}", + headers={ + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.3c6 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36" + }, + ).text + )["count"] + + def featured_data(self): + """ + Returns: + dict: Gets info on the user's featured project and featured label (like "Featured project", "My favorite things", etc.) + """ + try: + response = requests.get(f"https://scratch.mit.edu/site-api/users/all/{self.username}/").json() + return { + "label": response["featured_project_label_name"], + "project": dict( + id=str(response["featured_project_data"]["id"]), + author=response["featured_project_data"]["creator"], + thumbnail_url="https://" + response["featured_project_data"]["thumbnail_url"][2:], + title=response["featured_project_data"]["title"], + ), + } + except Exception: + return None + + def unfollowers(self) -> list[User]: + """ + Get all unfollowers by comparing API response and HTML response. + NOTE: This method can take a long time to run. + + Based on https://juegostrower.github.io/unfollowers/ + """ + follower_count = self.follower_count() + usernames = [] + for i in range(1, 2 + follower_count // 60): + with requests.no_error_handling(): + resp = requests.get(f"https://scratch.mit.edu/users/{self.username}/followers/", params={"page": i}) + soup = BeautifulSoup(resp.text, "html.parser") + usernames.extend((span.text.strip() for span in soup.select("span.title"))) + unfollowers = [] + for offset in range(0, follower_count, 40): + unfollowers.extend((user for user in self.followers(offset=offset, limit=40) if user.username not in usernames)) + return unfollowers + + def unfollower_usernames(self) -> list[str]: + return [user.username for user in self.unfollowers()] + + def follower_count(self): + with requests.no_error_handling(): + text = requests.get(f"https://scratch.mit.edu/users/{self.username}/followers/", headers=self._headers).text + return commons.webscrape_count(text, "Followers (", ")") + + def following_count(self): + with requests.no_error_handling(): + text = requests.get(f"https://scratch.mit.edu/users/{self.username}/following/", headers=self._headers).text + return commons.webscrape_count(text, "Following (", ")") + + def followers(self, *, limit=40, offset=0): + """ + Returns: + list: The user's followers as list of scratchattach.user.User objects + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/followers/", limit=limit, offset=offset + ) + return commons.parse_object_list(response, User, self._session, "username") + + def follower_names(self, *, limit=40, offset=0): + """ + Returns: + list: The usernames of the user's followers + """ + return [i.name for i in self.followers(limit=limit, offset=offset)] + + def following(self, *, limit=40, offset=0): + """ + Returns: + list: The users that the user is following as list of scratchattach.user.User objects + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/following/", limit=limit, offset=offset + ) + return commons.parse_object_list(response, User, self._session, "username") + + def following_names(self, *, limit=40, offset=0): + """ + Returns: + list: The usernames of the users the user is following + """ + return [i.name for i in self.following(limit=limit, offset=offset)] + + def is_following(self, user: str): + """ + Returns: + boolean: Whether the user is following the user provided as argument + """ + offset = 0 + following = False + while True: + try: + following_names = self.following_names(limit=20, offset=offset) + if user in following_names: + following = True + break + if not following_names: + break + offset += 20 + except Exception as e: + print(f"Warning: API error when performing following check: e={e!r}") + return following + return following + + def is_followed_by(self, user): + """ + Returns: + boolean: Whether the user is followed by the user provided as argument + """ + offset = 0 + followed = False + while True: + try: + followed_names = self.follower_names(limit=20, offset=offset) + if user in followed_names: + followed = True + break + if not followed_names: + break + offset += 20 + except Exception as e: + print(f"Warning: API error when performing following check: e={e!r}") + return followed + return followed + + def is_followed_by_me(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Returns: + boolean: Whether the user is followed by the user currently logged in. + """ + self._assert_auth() + with requests.no_error_handling(): + resp = requests.get( + f"https://scratch.mit.edu/users/{self.username}/", headers=self._headers, cookies=self._cookies + ) + soup = BeautifulSoup(resp.text, "html.parser") + follow_btn = soup.select_one("div.follow-button") + if not follow_btn: + print("Warning: follow button not found in page.") + return False + data_control = follow_btn.get("data-control") + return data_control == "unfollow" + + def project_count(self): + with requests.no_error_handling(): + text = requests.get(f"https://scratch.mit.edu/users/{self.username}/projects/", headers=self._headers).text + return commons.webscrape_count(text, "Shared Projects (", ")") + + def studio_count(self): + with requests.no_error_handling(): + text = requests.get(f"https://scratch.mit.edu/users/{self.username}/studios/", headers=self._headers).text + return commons.webscrape_count(text, "Studios I Curate (", ")") + + def studios_following_count(self): + with requests.no_error_handling(): + text = requests.get( + f"https://scratch.mit.edu/users/{self.username}/studios_following/", headers=self._headers + ).text + return commons.webscrape_count(text, "Studios I Follow (", ")") + + def studios(self, *, limit=40, offset=0) -> list[studio.Studio]: + _studios = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/studios/curate", limit=limit, offset=offset + ) + studios = [] + for studio_dict in _studios: + _studio = studio.Studio(_session=self._session, id=studio_dict["id"]) + _studio._update_from_data(studio_dict) + studios.append(_studio) + return studios + + def studios_following(self) -> list[studio.Studio]: + with requests.no_error_handling(): + resp = requests.get(f"https://scratch.mit.edu/users/{self.username}/studios_following/", headers=self._headers) + soup = BeautifulSoup(resp.text, "html.parser") + grid = soup.select_one(".media-grid") + assert grid is not None + studios: list[studio.Studio] = [] + for studio_elem in grid.select("li.gallery.thumb.item"): + title_span = studio_elem.select_one("span.title") + assert title_span is not None + anchor = title_span.find("a") + assert anchor is not None + href = str(anchor["href"]) + sid = int(href.split("/")[-2]) + title: str = anchor.text + if "\n" in title: + title = title.split("\n")[0] + studios.append(studio.Studio(id=sid, title=title, _session=self._session)) + return studios + + def projects(self, *, limit=40, offset=0) -> list[project.Project]: + """ + Returns: + list: The user's shared projects + """ + _projects = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/projects/", limit=limit, offset=offset, _headers=self._headers + ) + for p in _projects: + p["author"] = {"username": self.username} + return commons.parse_object_list(_projects, project.Project, self._session) + + def loves(self, *, limit=40, offset=0, get_full_project: bool = False) -> list[project.Project]: + """ + Returns: + list: The user's loved projects + """ + if offset < 0: + raise exceptions.BadRequest("offset parameter must be >= 0") + if limit < 0: + raise exceptions.BadRequest("limit parameter must be >= 0") + pages = range(1 + offset // 40, 2 + (offset + limit - 1) // 40) + _projects = [] + for page in pages: + first_idx = (page - 1) * 40 + with requests.no_error_handling(): + page_content = requests.get( + f"https://scratch.mit.edu/projects/all/{self.username}/loves/?page={page}", headers=self._headers + ).content + soup = BeautifulSoup(page_content, "html.parser") + h1_tag = soup.find("h1") + if h1_tag is not None: + if "Whoops! Our server is Scratch'ing its head" in h1_tag.text: + break + for i, project_element in enumerate(soup.find_all("li", {"class": "project thumb item"})): + if offset <= first_idx + i <= offset + limit: + assert isinstance(project_element, Tag) + project_anchors = project_element.find_all("a") + first_anchor = project_anchors[0] + second_anchor = project_anchors[1] + third_anchor = project_anchors[2] + assert isinstance(first_anchor, Tag) + assert isinstance(second_anchor, Tag) + assert isinstance(third_anchor, Tag) + project_id = commons.webscrape_count(first_anchor.attrs["href"], "/projects/", "/") + title = second_anchor.text + author = third_anchor.contents[0] + _project = project.Project( + id=project_id, + _session=self._session, + title=title, + author_name=author, + url=f"https://scratch.mit.edu/projects/{project_id}/", + ) + if get_full_project: + _project.update() + _projects.append(_project) + return _projects + + def loves_count(self): + with requests.no_error_handling(): + text = requests.get(f"https://scratch.mit.edu/projects/all/{self.username}/loves/", headers=self._headers).text + soup = BeautifulSoup(text, "html.parser") + if not soup.find("li", {"class": "project thumb item"}): + return 0 + return commons.webscrape_count(text, "»\n\n (", ")") + + def favorites(self, *, limit=40, offset=0): + """ + Returns: + list: The user's favorite projects + """ + _projects = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/favorites/", limit=limit, offset=offset, _headers=self._headers + ) + return commons.parse_object_list(_projects, project.Project, self._session) + + def favorites_count(self): + with requests.no_error_handling(): + text = requests.get(f"https://scratch.mit.edu/users/{self.username}/favorites/", headers=self._headers).text + return commons.webscrape_count(text, "Favorites (", ")") + + def has_badge(self) -> bool: + """ + Returns: + bool: Whether the user has a scratch membership badge on their profile (located next to the follow button) + """ + with requests.no_error_handling(): + resp = requests.get(self.url) + soup = BeautifulSoup(resp.text, "html.parser") + head = soup.find("div", {"class": "box-head"}) + if not head: + return False + for child in head.children: + if child.name == "img": + if "membership-badge.svg" in child["src"]: + return True + return False + + def toggle_commenting(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_permission() + requests.post( + f"https://scratch.mit.edu/site-api/comments/user/{self.username}/toggle-comments/", + headers=headers, + cookies=self._cookies, + ) + + def viewed_projects(self, limit=24, offset=0): + """ + Returns: + list: The user's recently viewed projects + + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_permission() + _projects = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/projects/recentlyviewed", + limit=limit, + offset=offset, + _headers=self._headers, + ) + return commons.parse_object_list(_projects, project.Project, self._session) + + def set_pfp(self, image: bytes): + """ + Sets the user's profile picture. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + requests.post( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", + headers=self._headers, + cookies=self._cookies, + files={"file": image}, + ) + + def set_bio(self, text): + """ + Sets the user's "About me" section. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + requests.put( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", + headers=self._json_headers, + cookies=self._cookies, + json={"bio": text}, + ) + + def set_wiwo(self, text): + """ + Sets the user's "What I'm working on" section. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + requests.put( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", + headers=self._json_headers, + cookies=self._cookies, + json={"status": text}, + ) + + def set_featured(self, project_id, *, label=""): + """ + Sets the user's featured project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Args: + project_id: Project id of the project that should be set as featured + + Keyword Args: + label: The label that should appear above the featured project on the user's profile (Like "Featured project", "Featured tutorial", "My favorite things", etc.) + """ + self._assert_permission() + requests.put( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", + headers=self._json_headers, + cookies=self._cookies, + json={"featured_project": int(project_id), "featured_project_label": label}, + ) + + def set_forum_signature(self, text): + """ + Sets the user's discuss forum signature. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_permission() + headers = { + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "content-type": "application/x-www-form-urlencoded", + "origin": "https://scratch.mit.edu", + "referer": "https://scratch.mit.edu/discuss/settings/TimMcCool/", + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", + } + data = {"csrfmiddlewaretoken": "a", "signature": text, "update": ""} + response = requests.post( + f"https://scratch.mit.edu/discuss/settings/{self.username}/", cookies=self._cookies, headers=headers, data=data + ) + + def post_comment(self, content, *, parent_id="", commentee_id=""): + """ + Posts a comment on the user's profile. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Args: + :param content: Content of the comment that should be posted + + Keyword Arguments: + :param commentee_id: ID of the comment you want to reply to. If you don't want to mention a user, don't put the argument. + :param parent_id: ID of the user that will be mentioned in your comment and will receive a message about your comment. If you don't want to mention a user, don't put the argument. + + Returns: + scratchattach.comment.Comment: An object representing the created comment. + """ + self._assert_auth() + data = {"commentee_id": commentee_id, "content": str(content), "parent_id": parent_id} + r = requests.post( + f"https://scratch.mit.edu/site-api/comments/user/{self.username}/add/", + headers=headers, + cookies=self._cookies, + data=json.dumps(data), + ) + if r.status_code != 200: + if "Looks like we are having issues with our servers!" in r.text: + raise exceptions.BadRequest("Invalid arguments passed") + else: + raise exceptions.CommentPostFailure(r.text) + text = r.text + try: + data = { + "id": text.split('
')[1].split("
")[0].strip(), + "reply_count": 0, + "cached_replies": [], + } + _comment = comment.Comment( + source=comment.CommentSource.USER_PROFILE, + parent_id=None if parent_id == "" else parent_id, + commentee_id=commentee_id, + source_id=self.username, + id=data["id"], + _session=self._session, + datetime=datetime.now(), + ) + _comment._update_from_data(data) + return _comment + except Exception as e: + if '{"error": "isFlood"}' in text: + raise exceptions.CommentPostFailure( + "You are being rate-limited for running this operation too often. Implement a cooldown of about 10 seconds." + ) from e + elif '")[0] + error_data = json.loads(raw_error_data) + expires = error_data["mute_status"]["muteExpiresAt"] + expires = datetime.fromtimestamp(expires, timezone.utc) + raise exceptions.CommentPostFailure(f"You have been muted. Mute expires on {expires}") from e + else: + raise exceptions.FetchError(f"Couldn't parse API response: {r.text!r}") from e + + def reply_comment(self, content, *, parent_id, commentee_id=""): + """ + Replies to a comment given by its id + + Warning: + Only replies to top-level comments are shown on the Scratch website. Replies to replies are actually replies to the corresponding top-level comment in the API. + + Therefore, parent_id should be the comment id of a top level comment. + + Args: + :param content: Content of the comment that should be posted + + Keyword Arguments: + :param parent_id: ID of the comment you want to reply to + :param commentee_id: ID of the user that will be mentioned in your comment and will receive a message about your comment. If you don't want to mention a user, don't put the argument. + """ + return self.post_comment(content, parent_id=parent_id, commentee_id=commentee_id) + + def activity(self, *, limit=1000): + """ + Returns: + list: The user's activity data as parsed list of scratchattach.activity.Activity objects + """ + with requests.no_error_handling(): + soup = BeautifulSoup( + requests.get(f"https://scratch.mit.edu/messages/ajax/user-activity/?user={self.username}&max={limit}").text, + "html.parser", + ) + activities = [] + source = soup.find_all("li") + for data in source: + _activity = activity.Activity(_session=self._session, raw=data) + _activity._update_from_html(data) + activities.append(_activity) + return activities + + def activity_html(self, *, limit=1000): + """ + Returns: + str: The raw user activity HTML data + """ + with requests.no_error_handling(): + return requests.get(f"https://scratch.mit.edu/messages/ajax/user-activity/?user={self.username}&max={limit}").text + + def follow(self): + """ + Follows the user represented by the User object. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/users/followers/{self.username}/add/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + ) + + def unfollow(self): + """ + Unfollows the user represented by the User object. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/users/followers/{self.username}/remove/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + ) + + def delete_comment(self, *, comment_id): + """ + Deletes a comment by its ID. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Args: + comment_id: The id of the comment that should be deleted + """ + self._assert_permission() + with requests.no_error_handling(): + return requests.post( + f"https://scratch.mit.edu/site-api/comments/user/{self.username}/del/", + headers=headers, + cookies=self._cookies, + data=json.dumps({"id": str(comment_id)}), + ) + + def report_comment(self, *, comment_id): + """ + Reports a comment by its ID to the Scratch team. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Args: + comment_id: The id of the comment that should be reported + """ + self._assert_auth() + return requests.post( + f"https://scratch.mit.edu/site-api/comments/user/{self.username}/rep/", + headers=headers, + cookies=self._cookies, + data=json.dumps({"id": str(comment_id)}), + ) + + def comments(self, *, page=1) -> list[comment.Comment] | None: + """ + Returns the comments posted on the user's profile (with replies). + + Keyword Arguments: + page: The page of the comments that should be returned. + + Returns: + list: A list containing the requested comments as Comment objects. + """ + data = [] + with requests.no_error_handling(): + resp = requests.get(f"https://scratch.mit.edu/site-api/comments/user/{self.username}/?page={page}") + if resp.status_code == 404: + return None + soup = BeautifulSoup(resp.content, "html.parser") + _comments = soup.find_all("li", {"class": "top-level-reply"}) + for entity in _comments: + comment_id = entity.find("div", {"class": "comment"})["data-comment-id"] + user = entity.find("a", {"id": "comment-user"})["data-comment-user"] + content = str(entity.find("div", {"class": "content"}).text).strip() + time = entity.find("span", {"class": "time"})["title"] + main_comment = {"id": comment_id, "author": {"username": user}, "content": content, "datetime_created": time} + _comment = comment.Comment( + source=comment.CommentSource.USER_PROFILE, source_id=self.username, _session=self._session + ) + _comment._update_from_data(main_comment) + reply_objs = [] + replies = entity.find_all("li", {"class": "reply"}) + for reply in replies: + r_comment_id = reply.find("div", {"class": "comment"})["data-comment-id"] + r_user = reply.find("a", {"id": "comment-user"})["data-comment-user"] + r_content = ( + str(reply.find("div", {"class": "content"}).text) + .strip() + .replace("\n", "") + .replace(" ", " ") + ) + r_time = reply.find("span", {"class": "time"})["title"] + reply_data = { + "id": r_comment_id, + "author": {"username": r_user}, + "content": r_content, + "datetime_created": r_time, + "parent_id": comment_id, + "cached_parent_comment": _comment, + } + _r_comment = comment.Comment( + source=comment.CommentSource.USER_PROFILE, + source_id=self.username, + _session=self._session, + cached_parent_comment=_comment, + ) + _r_comment._update_from_data(reply_data) + reply_objs.append(_r_comment) + _comment.reply_count = len(reply_objs) + _comment.cached_replies = list(reply_objs) + data.append(_comment) + return data + + def comment_by_id(self, comment_id) -> comment.Comment: + """ + Gets a comment on this user's profile by id. + + Warning: + For comments very far down on the user's profile, this method will take a while to find the comment. Very old comment are deleted from Scratch's database and may not appear. + + Returns: + scratchattach.comments.Comment: The request comment. + """ + page = 1 + page_content = self.comments(page=page) + while page_content != []: + results = list(filter(lambda x: str(x.id) == str(comment_id), page_content)) + if results == []: + results = list( + filter(lambda x: str(x.id) == str(comment_id), [item for x in page_content for item in x.cached_replies]) + ) + if results != []: + return results[0] + else: + return results[0] + page += 1 + page_content = self.comments(page=page) + raise exceptions.CommentNotFound() + + def message_events(self): + return message_events.MessageEvents(self) + + @deprecated("This method is deprecated because ScratchDB is down indefinitely.") + def stats(self): + """ + Gets information about the user's stats. Fetched from ScratchDB. + + Warning: + ScratchDB is down indefinitely, therefore this method is deprecated. + + Returns: + dict: A dict containing the user's stats. If the stats aren't available, all values will be -1. + """ + try: + stats = requests.get(f"https://scratchdb.lefty.one/v3/user/info/{self.username}").json()["statistics"] + stats.pop("ranks") + except Exception: + stats = {"loves": -1, "favorites": -1, "comments": -1, "views": -1, "followers": -1, "following": -1} + return stats + + @deprecated("Warning: ScratchDB is down indefinitely, therefore this method is deprecated.") + def ranks(self): + """ + Gets information about the user's ranks. Fetched from ScratchDB. + + Warning: + ScratchDB is down indefinitely, therefore this method is deprecated. + + Returns: + dict: A dict containing the user's ranks. If the ranks aren't available, all values will be -1. + """ + try: + return requests.get(f"https://scratchdb.lefty.one/v3/user/info/{self.username}").json()["statistics"]["ranks"] + except Exception: + return { + "country": {"loves": 0, "favorites": 0, "comments": 0, "views": 0, "followers": 0, "following": 0}, + "loves": 0, + "favorites": 0, + "comments": 0, + "views": 0, + "followers": 0, + "following": 0, + } + + def ocular_status(self) -> _OcularStatus: + """ + Gets information about the user's ocular status. Ocular is a website developed by jeffalo: https://ocular.jeffalo.net/ + + Returns: + dict + """ + return requests.get(f"https://my-ocular.jeffalo.net/api/user/{self.username}").json() + + def verify_identity(self, *, verification_project_id=395330233): + """ + Can be used in applications to verify a user's identity. + + This function returns a Verifactor object. Attributs of this object: + :.projecturl: The link to the project where the user has to go to verify + :.project: The project where the user has to go to verify as scratchattach.Project object + :.code: The code that the user has to comment + + To check if the user verified successfully, call the .check() function on the returned object. + It will return True if the user commented the code. + """ + v = Verificator(self, verification_project_id) + return v + + def rank(self) -> Rank: + """ + Finds the rank of the user. + Returns a member of the Rank enum: either Rank.NEW_SCRATCHER, Rank.SCRATCHER, or Rank.SCRATCH_TEAM. + May replace user.scratchteam and user.is_new_scratcher in the future. + """ + if self.is_new_scratcher(): + return Rank.NEW_SCRATCHER + if not self.scratchteam: + return Rank.SCRATCHER + return Rank.SCRATCH_TEAM + + +def get_user(username) -> User: + """ + Gets a user without logging in. + + Args: + username (str): Username of the requested user + + Returns: + scratchattach.user.User: An object representing the requested user + + Warning: + Any methods that require authentication (like user.follow) will not work on the returned object. + + If you want to use these, get the user with :meth:`scratchattach.session.Session.connect_user` instead. + """ + warnings.warn( + "Warning: For methods that require authentication, use session.connect_user instead of get_user.\nTo ignore this warning, use warnings.filterwarnings('ignore', category=scratchattach.UserAuthenticationWarning).\nTo ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use `warnings.filterwarnings('ignore', category=scratchattach.GetAuthenticationWarning)`.", + exceptions.UserAuthenticationWarning, + ) + return commons._get_object("username", username, User, exceptions.UserNotFound) diff --git a/scratchattach/eventhandlers/cloud_server.py b/scratchattach/eventhandlers/cloud_server.py index d950f56f..922236b0 100644 --- a/scratchattach/eventhandlers/cloud_server.py +++ b/scratchattach/eventhandlers/cloud_server.py @@ -200,10 +200,7 @@ def __init__( blocked_ips = [] SimpleWebSocketServer.__init__(self, hostname, port=port, websocketclass=websocketclass) - BaseEventHandler.__init__(self) - - self.running = False - self._events = {} # saves event functions called on cloud updates + BaseEventHandler.__init__(self) # saves event functions called on cloud updates self.tw_clients = {} # saves connected clients self.tw_variables = {} # holds cloud variable states diff --git a/scratchattach/site/session.py b/scratchattach/site/session.py index bf182994..4cc8e795 100644 --- a/scratchattach/site/session.py +++ b/scratchattach/site/session.py @@ -906,13 +906,13 @@ def mystuff_studios( def mystuff_classes_counts(self) -> tuple[int, int]: """ Returns the number of open and ended classes owned by a teacher session. - If this is not a teacher session, (0, 0) is returned. + If this is not a teacher session, NotATeacherError is raised """ with requests.no_error_handling(): - resp = requests.get("https://scratch.mit.edu/educators/classes/") + resp = requests.get("https://scratch.mit.edu/educators/classes/", headers=self._headers, cookies=self._cookies) if resp.status_code == 403: - return 0, 0 # non-teacher account + raise exceptions.NotATeacherError("Response 403 when getting educators/classes") soup = BeautifulSoup(resp.text, "html.parser") sidebar = soup.find("div", {"id": "sidebar", "class": "tabs-index"}) @@ -1495,5 +1495,6 @@ def login_from_browser(browser: Browser = ANY): """ cookies = cookies_from_browser(browser) if "scratchsessionsid" in cookies: - return login_by_id(cookies["scratchsessionsid"]) + with suppress_login_warning(): + return login_by_id(cookies["scratchsessionsid"]) raise ValueError("Not enough data to log in.") diff --git a/scratchattach/sync_api/primitives/http.py b/scratchattach/sync_api/primitives/http.py new file mode 100644 index 00000000..3abd6b6c --- /dev/null +++ b/scratchattach/sync_api/primitives/http.py @@ -0,0 +1,235 @@ +from __future__ import annotations +from types import TracebackType +from collections.abc import Iterable, Mapping +from typing import Optional, Self, cast, Any, Sequence, SupportsInt, BinaryIO, TYPE_CHECKING + +if TYPE_CHECKING: + from _typeshed import SupportsKeysAndGetItem +from scratchattach._shared import http as shared_http +import requests +from requests import cookies as requests_cookies + +HTTPOptions = shared_http.HTTPOptions + + +class _HTTPResponse: + _sync_response: requests.Response + + def text(self) -> str: + return self._sync_response.text + + def content(self) -> bytes: + return self._sync_response.content + + def json(self) -> Any: + return self._sync_response.json() + + @property + def headers(self) -> Mapping[str, str]: + """ + Headers are case-insensitive. + """ + return self._sync_response.headers + + def get_all_headers_for_key(self, key: str) -> list[str]: + return self._sync_response.raw.headers.getlist(key) + + @property + def status_code(self) -> int: + return self._sync_response.status_code + + +class _WrappedHTTPResponse: + _response: requests.Response + + def __init__(self, response: requests.Response): + self._response = response + + def __enter__(self) -> _HTTPResponse: + response = _HTTPResponse() + response._sync_response = self._response + return response + + def __exit__( + self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] + ) -> None: + self._response.close() + + +class DummyCookieJar(requests_cookies.RequestsCookieJar): + def set_cookie(self, *args, **kwargs): + pass + + def update(self, *args, **kwargs): + pass + + def copy(self): + return DummyCookieJar() + + def __setitem__(self, name, value): + pass + + def set(self, *args, **kwargs): + pass + + +class _HTTPSession: + def add_cookie(self, key: str, value: str): + self._cookies[key] = value + + def get_cookie(self, key: str) -> Optional[str]: + return self._cookies.get(key) + + def remove_cookie(self, key: str): + del self._cookies[key] + + def clear_cookies(self): + self._cookies.clear() + + def update_cookies(self, new: "SupportsKeysAndGetItem[str, str]"): + self._cookies.update(new) + + def list_cookies(self) -> Iterable[tuple[str, str]]: + return self._cookies.items() + + def add_header(self, key: str, value: str): + self._headers[key] = value + + def get_header(self, key: str) -> Optional[str]: + return self._headers.get(key) + + def remove_header(self, key: str): + del self._headers[key] + + def clear_headers(self): + self._headers.clear() + + def update_headers(self, new: "SupportsKeysAndGetItem[str, str]"): + self._headers.update(new) + + def list_headers(self) -> Iterable[tuple[str, str]]: + return self._headers.items() + + _cookies: dict[str, str] + _headers: dict[str, str] + _http_session: requests.Session + + def __init__(self): + self._cookies = {} + self._headers = {} + self._http_session = requests.Session() + self._http_session.cookies = DummyCookieJar() + + def __enter__(self) -> Self: + return self + + def __exit__( + self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] + ) -> None: + pass + + @staticmethod + def _get_params_kwarg(params: Any) -> Any: + if isinstance(params, shared_http.SupportsItems): + params = params.items() + if not isinstance(params, str) and isinstance(params, Iterable): + new_params: Any = [] + for key, value in cast(Iterable[tuple[str, Iterable[str | SupportsInt] | SupportsInt | None]], params): + if not isinstance(value, str) and isinstance(value, Iterable): + for item in value: + new_params.append((key, cast(str | SupportsInt, item))) + elif value is None: + pass + else: + new_params.append((key, cast(str | SupportsInt, value))) + params = new_params + return params + + @staticmethod + def _get_data_and_files_kwargs( + data: Iterable[tuple[str, Any]] | shared_http.SupportsItems[str, Any] | None, + files: Iterable[tuple[str, BinaryIO | bytes]] | shared_http.SupportsItems[str, BinaryIO | bytes] | None, + ) -> tuple[list[tuple[str, Any]] | None, list[tuple[str, BinaryIO | bytes]] | None]: + processed_data = None + processed_files = None + if data is not None: + if isinstance(data, shared_http.SupportsItems): + data = cast(Any, data.items()) + processed_data = list(cast(Iterable[tuple[str, Any]], data)) + if files is not None: + if isinstance(files, shared_http.SupportsItems): + files = cast(Any, files.items()) + processed_files = list(cast(Iterable[tuple[str, BinaryIO | bytes]], files)) + return (processed_data, processed_files) + + @staticmethod + def _get_headers_kwarg( + default_headers: dict[str, str], headers: Iterable[tuple[str, str]] | shared_http.SupportsItems[str, str] | None + ) -> dict[str, str]: + if headers is None: + return default_headers + if isinstance(headers, shared_http.SupportsItems): + headers = cast(Iterable[tuple[str, str]], headers.items()) + return default_headers | dict(headers) + + @staticmethod + def _get_cookies_kwarg( + default_cookies: dict[str, str], cookies: Iterable[tuple[str, str]] | shared_http.SupportsItems[str, str] | None + ) -> dict[str, str]: + if cookies is None: + return default_cookies + if isinstance(cookies, shared_http.SupportsItems): + cookies = cast(Iterable[tuple[str, str]], cookies.items()) + return default_cookies | dict(cookies) + + def _get_kwargs(self, options: HTTPOptions) -> dict[str, Any]: # noqa: C901 + kwargs: dict[str, Any] = {} + if options.params is not None: + kwargs["params"] = self._get_params_kwarg(options.params) + if options.content is not None and options.data is not None: + raise ValueError('Cannot specify both "content" and "data"') + if options.content is not None and options.files is not None: + raise ValueError('Cannot specify both "content" and "files"') + if options.content is not None: + kwargs["data"] = options.content + if options.data is not None or options.files is not None: + processed_data, processed_files = self._get_data_and_files_kwargs(options.data, options.files) + if processed_data is not None: + kwargs["data"] = processed_data + if processed_files is not None: + kwargs["files"] = processed_files + kwargs["cookies"] = self._get_cookies_kwarg( + {} if options.disregard_default_cookies else self._cookies, options.cookies + ) + kwargs["headers"] = self._get_headers_kwarg( + {} if options.disregard_default_headers else self._headers, options.headers + ) + if options.json is not shared_http._JsonEmptySentinel and ( + options.content is not None or options.data is not None or options.files is not None + ): + raise ValueError('Cannot specify "json" alongside "content", "data", or "files"') + if options.json is not shared_http._JsonEmptySentinel: + kwargs["json"] = options.json + if options.timeout: + kwargs["timeout"] = options.timeout + return kwargs + + def get(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs(options if options is not None else shared_http._EMPTY_OPTIONS) + return _WrappedHTTPResponse(self._http_session.get(url, **kwargs)) + + def post(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs(options if options is not None else shared_http._EMPTY_OPTIONS) + return _WrappedHTTPResponse(self._http_session.post(url, **kwargs)) + + def put(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs(options if options is not None else shared_http._EMPTY_OPTIONS) + return _WrappedHTTPResponse(self._http_session.put(url, **kwargs)) + + def delete(self, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs(options if options is not None else shared_http._EMPTY_OPTIONS) + return _WrappedHTTPResponse(self._http_session.delete(url, **kwargs)) + + def request(self, method: shared_http.HTTPMethod, url: str, options: HTTPOptions | None = None) -> _WrappedHTTPResponse: + kwargs = self._get_kwargs(options if options is not None else shared_http._EMPTY_OPTIONS) + return _WrappedHTTPResponse(self._http_session.request(method.name, url, **kwargs)) diff --git a/scratchattach/sync_api/primitives/utils.py b/scratchattach/sync_api/primitives/utils.py new file mode 100644 index 00000000..ed119d3a --- /dev/null +++ b/scratchattach/sync_api/primitives/utils.py @@ -0,0 +1,132 @@ +from __future__ import annotations +from collections.abc import Callable +from typing import Union, ParamSpec, TypeVar, Generic, Any, cast, Optional, overload, Literal +import time + +try: + import ctypes + + CTYPES_PRESENT = True +except Exception: + CTYPES_PRESENT = False +import threading +import concurrent.futures + + +def sleep_prim(delay: Union[int, float]): + time.sleep(delay) + + +P = ParamSpec("P") +O = TypeVar("O", covariant=True) + + +class Task(Generic[P, O]): + function: Callable[P, O] + args: Any + kwargs: Any + available: bool + + +class LaunchedTask(Generic[P, O]): + task: Task[P, O] + _out: O + _thread: threading.Thread + + +def create_task(function: Callable[P, O], *args: P.args, **kwargs: P.kwargs) -> Task[P, O]: + task: Task[P, O] = Task() + task.function = function + task.args = args + task.kwargs = kwargs + task.available = True + return task + + +def gather_concurrently_prim(*tasks: Task[Any, O]) -> list[O]: + with concurrent.futures.ThreadPoolExecutor() as executor: + return [cast(O, i) for i in executor.map(lambda x: x.function(*x.args, **x.kwargs), tasks)] + + +def launch_concurrently_prim(task: Task[P, O]) -> LaunchedTask[P, O]: + launched_task: LaunchedTask[P, O] = LaunchedTask() + + def wrap_function(): + launched_task._out = task.function(*task.args, **task.kwargs) + + thread = threading.Thread(target=wrap_function) + thread.start() + launched_task.task = task + launched_task._thread = thread + return launched_task + + +A = TypeVar("A") +B = TypeVar("B") + + +@overload +def join_launched_task_prim(task: LaunchedTask[P, O]) -> O: + pass + + +@overload +def join_launched_task_prim(task: LaunchedTask[P, O], timeout: Union[float, int]) -> Optional[O]: + pass + + +def join_launched_task_prim(task: LaunchedTask[P, O], timeout: Optional[Union[float, int]] = None) -> Optional[O]: + task._thread.join(timeout) + if task._thread.is_alive(): + return None + return task._out + + +def _raise_in_thread(thread: threading.Thread, exc_type: type[BaseException]) -> None: + if not CTYPES_PRESENT: + raise NotImplementedError("Sending exceptions to threads is not supported in this Python version.") + if not thread.is_alive(): + raise ValueError("Thread is not alive.") + thread_id = thread.ident + if thread_id is None: + raise ValueError("Thread has no ident.") + result = ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_ulong(thread_id), ctypes.py_object(exc_type)) + if result == 0: + raise ValueError("Thread ident is invalid.") + if result > 1: + ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_ulong(thread_id), None) + raise SystemError("PyThreadState_SetAsyncExc failed.") + + +@overload +def kill_launched_task_prim(task: LaunchedTask[P, O], *, exception_interval: Union[float, int] = 0.1) -> Literal[True]: + """ + Sends exceptions to the underlying concurrency primitive. + May also try to use the recommended way of cancelling the primitive if there is one. + Returns whether the task was actually killed. + """ + + +@overload +def kill_launched_task_prim( + task: LaunchedTask[P, O], timeout: Union[float, int], *, exception_interval: Union[float, int] = 0.1 +) -> bool: + """ + Sends exceptions to the underlying concurrency primitive. + May also try to use the recommended way of cancelling the primitive if there is one. + Returns whether the task was actually killed. + """ + + +def kill_launched_task_prim( + task: LaunchedTask[P, O], timeout: Optional[Union[float, int]] = None, *, exception_interval: Union[float, int] = 0.1 +) -> bool: + has_timeout, timeout_end = (True, time.time() + timeout) if timeout is not None else (False, None) + while not has_timeout or (timeout_end is not None and time.time() <= timeout_end): + if not task._thread.is_alive(): + break + _raise_in_thread(task._thread, SystemExit) + time.sleep(exception_interval) + if has_timeout and timeout_end is not None and (time.time() > timeout_end): + return False + return True diff --git a/scratchattach/sync_api/site/_base.py b/scratchattach/sync_api/site/_base.py new file mode 100644 index 00000000..7256114b --- /dev/null +++ b/scratchattach/sync_api/site/_base.py @@ -0,0 +1,196 @@ +from __future__ import annotations +from abc import ABC, abstractmethod +from typing import TypeVar, Optional, Self, Union, Any, Generic, TypeAlias, cast, overload, Literal +import json +from collections.abc import Callable +from scratchattach.utils import exceptions, commons, optional_async +from scratchattach._shared import http as shared_http +from . import session +from ..primitives import http + +D = TypeVar("D") +C = TypeVar("C", bound="BaseSiteComponent") + + +class BaseSiteComponent(ABC, Generic[D]): + _session: session.Session | session.UnauthSession + update_api: str + + def update(self): + """ + Updates the attributes of the object by performing an API response. Returns True if the update was successful. + """ + with self._session.http_session.request( + self.update_method, self.update_api, shared_http.options().timeout(10).value + ) as response: + if response.status_code == 429: + return "429" + response_json = response.json() + if response_json == {"response": "Too many requests"}: + return "429" + assert isinstance(response_json, dict) + if "code" in response_json: + return False + return self._update_from_data(cast(D, response_json)) + + def updated(self) -> Self: + self.update() + return self + + @abstractmethod + def _update_from_data(self, data: D) -> bool: + """ + Parses the API response that is fetched in the update-method. Class specific, must be overridden in classes inheriting from this one. + """ + + def _assert_auth(self) -> session.Session: + if isinstance(self._session, session.UnauthSession): + raise exceptions.Unauthenticated( + "You need to use session.connect_xyz (NOT get_xyz) in order to perform this operation." + ) + return self._session + + @classmethod + def _get_object( + cls, identificator_name, identificator, NotFoundException, session: session.Session | session.UnauthSession + ) -> Self: + from scratchattach.site import project + + try: + use_class: type = cls + if cls is project.PartialProject: + use_class = project.Project + assert issubclass(use_class, cls) + _object = use_class(**{identificator_name: identificator, "_session": session}) + r = _object.update() + if r == "429": + raise exceptions.Response429( + "Your network is blocked or rate-limited by Scratch.\nIf you're using an online IDE like replit.com, try running the code on your computer." + ) + if not r: + if cls is project.PartialProject: + _object = project.PartialProject( + **{identificator_name: identificator, "shared": False, "_session": session} + ) + assert isinstance(_object, cls) + return _object + else: + raise NotFoundException + else: + return _object + except KeyError as e: + raise NotFoundException(f"Key error at key {e} when reading API response") + except Exception as e: + raise e + + def _make_linked_object(self, identificator_id, identificator, cls: type[C], not_found_exception) -> C: + """ + Internal function for making a linked object (authentication kept) based on an identificator (like a project id or username) + Class must inherit from BaseSiteComponent + """ + return cls._get_object(identificator_id, identificator, not_found_exception, self._session) + + @classmethod + def parse_object_list( + cls, raw: list[D], /, session: session.Session | session.UnauthSession, primary_key: str = "id" + ) -> list[Self]: + results = [] + for raw_dict in raw: + try: + _obj = cls(**{primary_key: cast(dict[str, Any], raw_dict)[primary_key], "_session": session}) + _obj._update_from_data(raw_dict) + results.append(_obj) + except Exception as e: + print("Warning raised by scratchattach: failed to parse ", raw_dict, "error", e) + return results + + def supply_data_dict(self, data: D) -> bool: + return self._update_from_data(data) + + update_method: shared_http.HTTPMethod = shared_http.HTTPMethod.GET + "\n HTTP method for getting updated information for this component\n " + + +F = TypeVar("F") + + +@overload +def api_iterative_data( + fetch_func: Callable[[int, int], list[F] | None], + limit: int, + offset: int, + max_req_limit: int = 40, + unpack: Literal[True] = True, +) -> list[F]: ... + + +@overload +def api_iterative_data( + fetch_func: Callable[[int, int], F | None], + limit: int, + offset: int, + max_req_limit: int = 40, + unpack: Literal[False] = False, +) -> list[F]: ... + + +def api_iterative_data( + fetch_func: Callable[[int, int], Any], limit: int, offset: int, max_req_limit: int = 40, unpack: bool = True +) -> list: + """ + Iteratively gets data by calling fetch_func with a moving offset and a limit. + Once fetch_func returns None, the retrieval is completed. + """ + if limit is None: + limit = max_req_limit + end = offset + limit + api_data: list[Any] = [] + for offs in range(offset, end, max_req_limit): + data = fetch_func(offs, max_req_limit) + if data is None: + break + if unpack: + api_data.extend(data) + else: + api_data.append(data) + if len(data) < max_req_limit: + break + return api_data[:limit] + + +def api_iterative( + session: session.Session | session.UnauthSession, + url: str, + *, + limit: int, + offset: int, + max_req_limit: int = 40, + add_params: str = "", + _headers: Optional[dict] = None, + cookies: Optional[dict] = None, +) -> list[F]: + """ + Function for getting data from one of Scratch's iterative JSON API endpoints (like /users//followers, or /users//projects) + """ + if _headers is None: + _headers = commons.headers.copy() + if cookies is None: + cookies = {} + if offset < 0: + raise exceptions.BadRequest("offset parameter must be >= 0") + if limit < 0: + raise exceptions.BadRequest("limit parameter must be >= 0") + + def fetch(off: int, lim: int) -> list[F] | None: + """ + Performs a single API request + """ + with session.http_session.get(f"{url}?limit={lim}&offset={off}{add_params}") as response: + resp = cast(list[F], response.json()) + if not resp: + return None + if resp == {"code": "BadRequest", "message": ""}: + raise exceptions.BadRequest("The arguments passed are invalid") + return resp + + return api_iterative_data(fetch, limit, offset, max_req_limit=max_req_limit, unpack=True) diff --git a/scratchattach/sync_api/site/activity.py b/scratchattach/sync_api/site/activity.py new file mode 100644 index 00000000..d762f62a --- /dev/null +++ b/scratchattach/sync_api/site/activity.py @@ -0,0 +1,372 @@ +"""Activity and CloudActivity class""" + +from __future__ import annotations +import html +import warnings +from dataclasses import dataclass +from typing import Optional, Any +from enum import Enum +from bs4 import Tag +from . import user, project, studio, session, forum, comment +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions + + +class ActivityTypes(Enum): + loveproject = "loveproject" + favoriteproject = "favoriteproject" + becomecurator = "becomecurator" + followuser = "followuser" + followstudio = "followstudio" + shareproject = "shareproject" + remixproject = "remixproject" + becomeownerstudio = "becomeownerstudio" + addcomment = "addcomment" + curatorinvite = "curatorinvite" + userjoin = "userjoin" + studioactivity = "studioactivity" + forumpost = "forumpost" + updatestudio = "updatestudio" + createstudio = "createstudio" + promotetomanager = "promotetomanager" + updateprofile = "updateprofile" + removeprojectfromstudio = "removeprojectfromstudio" + addprojecttostudio = "addprojecttostudio" + performaction = "performaction" + + +@dataclass +class Activity(BaseSiteComponent): + """ + Represents a Scratch activity (message or other user page activity) + """ + + _session: Optional[session.Session] = None + raw: Any = None + id: Optional[int] = None + actor_username: Optional[str] = None + project_id: Optional[int] = None + gallery_id: Optional[int] = None + username: Optional[str] = None + followed_username: Optional[str] = None + recipient_username: Optional[str] = None + title: Optional[str] = None + project_title: Optional[str] = None + gallery_title: Optional[str] = None + topic_title: Optional[str] = None + topic_id: Optional[int] = None + target_name: Optional[str] = None + target_id: Optional[int | str] = None + parent_title: Optional[str] = None + parent_id: Optional[int] = None + comment_type: Optional[int] = None + comment_obj_id: Optional[int] = None + comment_obj_title: Optional[str] = None + comment_id: Optional[int] = None + comment_fragment: Optional[str] = None + changed_fields: Optional[dict[str, str]] = None + is_reshare: Optional[bool] = None + datetime_created: Optional[str] = None + time: Any = None + type: Optional[ActivityTypes] = None + + def __repr__(self): + return f"Activity({repr(self.raw)})" + + def __str__(self): + return "-A " + " ".join(self.parts) + + def _parts_simple(self, verb: str, obj: str): + return [str(self.actor_username), verb, obj] + + def _parts_comment(self) -> list[str]: + ret = [str(self.actor_username), "commented on"] + if self.comment_type not in (0, 1, 2): + raise ValueError(f"Unknown comment type: {self.comment_type}") + ret.append( + { + 0: f"-P {self.comment_obj_title!r} ({self.comment_obj_id}", + 1: f"-U {self.comment_obj_title}", + 2: f"-S {self.comment_obj_title!r} ({self.comment_obj_id}", + }[self.comment_type] + ) + ret[-1] += f"#{self.comment_id})" + ret.append(str(html.unescape(str(self.comment_fragment)))) + return ret + + @property + def parts(self): + """ + Return format: [actor username] + N * [action, object] + :return: A list of parts of the message. Join the parts to get a readable version, which is done with str(activity) + """ + SIMPLE_SOLNS = { + ActivityTypes.loveproject: ("loved", f"-P {self.title!r} ({self.project_id})"), + ActivityTypes.favoriteproject: ("favorited", f"-P {self.project_title!r} ({self.project_id})"), + ActivityTypes.becomecurator: ("now curating", f"-S {self.title!r} ({self.gallery_id})"), + ActivityTypes.followuser: ("followed", f"-U {self.followed_username}"), + ActivityTypes.followstudio: ("followed", f"-S {self.title!r} ({self.gallery_id})"), + ActivityTypes.shareproject: ( + "reshared" if self.is_reshare else "shared", + f"-P {self.title!r} ({self.project_id})", + ), + ActivityTypes.remixproject: ( + "remixed", + f"-P {self.parent_title!r} ({self.parent_id}) as -P {self.title!r} ({self.project_id})", + ), + ActivityTypes.becomeownerstudio: ("became owner of", f"-S {self.gallery_title!r} ({self.gallery_id})"), + ActivityTypes.curatorinvite: ("invited you to curate", f"-S {self.title!r} ({self.gallery_id})"), + ActivityTypes.forumpost: ("posted in", f"-F {self.topic_title} ({self.topic_id})"), + ActivityTypes.updatestudio: ("updated", f"-S {self.gallery_title} ({self.gallery_id})"), + ActivityTypes.createstudio: ("created", f"-S {self.gallery_title} ({self.gallery_id})"), + None: (), + } + if args := SIMPLE_SOLNS.get(self.type): + return self._parts_simple(*args) + match self.type: + case ActivityTypes.addcomment: + return self._parts_comment() + case ActivityTypes.userjoin: + return [str(self.actor_username), "joined Scratch"] + case ActivityTypes.studioactivity: + return [str(self.actor_username), "Studio activity", "", f"-S {self.title!r} ({self.gallery_id})"] + case ActivityTypes.promotetomanager: + return [ + str(self.actor_username), + "promoted", + f"-U {self.recipient_username}", + "in", + f"-S {self.gallery_title} ({self.gallery_id})", + ] + case ActivityTypes.updateprofile: + return [str(self.actor_username), "updated their profile.", f"Changed fields: {self.changed_fields}"] + case ActivityTypes.removeprojectfromstudio: + return [ + f"{self.actor_username}", + "removed", + f"-P {self.project_title} ({self.project_id})", + "from", + f"-S {self.gallery_title} ({self.gallery_id})", + ] + case ActivityTypes.addprojecttostudio: + return [ + f"{self.actor_username}", + "added", + f"-P {self.project_title} ({self.project_id})", + "to", + f"-S {self.gallery_title} ({self.gallery_id})", + ] + case ActivityTypes.performaction: + return [f"{self.actor_username}", "performed an action"] + case _: + raise NotImplementedError( + f"Activity type {self.type!r} is not implemented!\nself.raw={self.raw!r}\nRaise an issue on github: https://github.com/TimMcCool/scratchattach/issues" + ) + + def update(self): + print("Warning: Activity objects can't be updated") + return False + + def _update_from_data(self, data): + self.raw = data + self._session = data.get("_session", self._session) + self.raw = data.get("raw", self.raw) + self.id = data.get("id", self.id) + self.actor_username = data.get("actor_username", self.actor_username) + self.project_id = data.get("project_id", self.project_id) + self.gallery_id = data.get("gallery_id", self.gallery_id) + self.username = data.get("username", self.username) + self.followed_username = data.get("followed_username", self.followed_username) + self.recipient_username = data.get("recipient_username", self.recipient_username) + self.title = data.get("title", self.title) + self.project_title = data.get("project_title", self.project_title) + self.gallery_title = data.get("gallery_title", self.gallery_title) + self.topic_title = data.get("topic_title", self.topic_title) + self.topic_id = data.get("topic_id", self.topic_id) + self.target_name = data.get("target_name", self.target_name) + self.target_id = data.get("target_id", self.target_id) + self.parent_title = data.get("parent_title", self.parent_title) + self.parent_id = data.get("parent_id", self.parent_id) + self.comment_type = data.get("comment_type", self.comment_type) + self.comment_obj_id = data.get("comment_obj_id", self.comment_obj_id) + self.comment_obj_title = data.get("comment_obj_title", self.comment_obj_title) + self.comment_id = data.get("comment_id", self.comment_id) + self.comment_fragment = data.get("comment_fragment", self.comment_fragment) + self.changed_fields = data.get("changed_fields", self.changed_fields) + self.is_reshare = data.get("is_reshare", self.is_reshare) + self.datetime_created = data.get("datetime_created", self.datetime_created) + self.time = data.get("time", self.time) + _type = data.get("type", self.type) + if _type == "becomehoststudio": + self.type = ActivityTypes.becomeownerstudio + elif _type: + self.type = ActivityTypes[_type] + return True + + def _update_from_json(self, data: dict): + """ + Update using JSON, used in the classroom API. + """ + activity_type = data["type"] + _time = data.get("datetime_created") + if "actor" in data: + self.username = data["actor"]["username"] + else: + self.username = data.get("actor_username") + self.recipient_username = None + if recipient := data.get("recipient"): + self.recipient_username = recipient["username"] + elif ru := data.get("recipient_username"): + self.recipient_username = ru + elif project_creator := data.get("project_creator"): + self.recipient_username = project_creator["username"] + self.actor_username = self.username + self.raw = data + self.datetime_created = _time + self.type = { + 0: ActivityTypes.followuser, + 1: ActivityTypes.followstudio, + 2: ActivityTypes.loveproject, + 3: ActivityTypes.favoriteproject, + 7: ActivityTypes.addprojecttostudio, + 8: ActivityTypes.shareproject, + 9: ActivityTypes.shareproject, + 10: ActivityTypes.shareproject, + 11: ActivityTypes.remixproject, + 13: ActivityTypes.createstudio, + 15: ActivityTypes.updatestudio, + 16: ActivityTypes.removeprojectfromstudio, + 17: ActivityTypes.removeprojectfromstudio, + 18: ActivityTypes.removeprojectfromstudio, + 19: ActivityTypes.removeprojectfromstudio, + 20: ActivityTypes.promotetomanager, + 21: ActivityTypes.promotetomanager, + 22: ActivityTypes.promotetomanager, + 23: ActivityTypes.updateprofile, + 24: ActivityTypes.updateprofile, + 25: ActivityTypes.updateprofile, + 26: ActivityTypes.addcomment, + 27: ActivityTypes.addcomment, + None: ActivityTypes.performaction, + }.get(activity_type, ActivityTypes.performaction) + self.followed_username = data.get("followed_username", self.followed_username) + self.gallery_id = data.get("gallery", self.gallery_id) + self.project_id = data.get("project", self.project_id) + self.is_reshare = data.get("is_reshare", self.is_reshare) + self.comment_fragment = data.get("comment_fragment", self.comment_fragment) + self.comment_type = data.get("comment_type", self.comment_type) + self.comment_obj_id = data.get("comment_obj_id", self.comment_obj_id) + self.comment_obj_title = data.get("comment_obj_title", self.comment_obj_title) + self.comment_id = data.get("comment_id", self.comment_id) + self.parent_id = data.get("parent", self.parent_id) + if self.parent_id: + warnings.warn( + f"This may be incorrectly implemented.\nRaw data: {data}\nPlease raise an issue on gh: https://github.com/TimMcCool/scratchattach/issues" + ) + if self.type == ActivityTypes.updateprofile: + self.changed_fields = data.get("changed_fields", {}) + + def _update_from_html(self, data: Tag): + self.raw = data + _time = data.find("div").find("span").find_next().find_next().text.strip() + if "\xa0" in _time: + while "\xa0" in _time: + _time = _time.replace("\xa0", " ") + self.datetime_created = _time + self.actor_username = data.find("div").find("span").text + self.target_name = data.find("div").find("span").find_next().text + self.target_link = data.find("div").find("span").find_next()["href"] + self.target_id = data.find("div").find("span").find_next()["href"].split("/")[-2] + _type = data.find("div").find_all("span")[0].next_sibling.strip() + if _type == "loved": + self.type = ActivityTypes.loveproject + elif _type == "favorited": + self.type = ActivityTypes.favoriteproject + elif "curator" in _type: + self.type = ActivityTypes.becomecurator + elif "shared" in _type: + self.type = ActivityTypes.shareproject + elif "is now following" in _type: + if "users" in self.target_link: + self.type = ActivityTypes.followuser + else: + self.type = ActivityTypes.followstudio + return True + + def actor(self): + """ + Returns the user that performed the activity as User object + """ + return self._make_linked_object("username", self.actor_username, user.User, exceptions.UserNotFound) + + def target_project(self) -> Optional[project.Project]: + if self.target_id: + return self._make_linked_object("id", self.target_id, project.Project, exceptions.ProjectNotFound) + if self.project_id: + return self._make_linked_object("id", self.project_id, project.Project, exceptions.ProjectNotFound) + return None + + def target_studio(self) -> Optional[studio.Studio]: + if self.target_id: + return self._make_linked_object("id", self.target_id, studio.Studio, exceptions.StudioNotFound) + if self.gallery_id: + return self._make_linked_object("id", self.gallery_id, studio.Studio, exceptions.StudioNotFound) + return None + + def target_user(self) -> Optional[user.User]: + if self.username: + return self._make_linked_object("username", self.username, user.User, exceptions.UserNotFound) + if self.target_name: + return self._make_linked_object("username", self.target_name, user.User, exceptions.UserNotFound) + if self.followed_username: + return self._make_linked_object("username", self.followed_username, user.User, exceptions.UserNotFound) + if self.recipient_username: + return self._make_linked_object("username", self.recipient_username, user.User, exceptions.UserNotFound) + return None + + def target_comment(self) -> Optional[comment.Comment]: + if self.comment_type == 0: + if self.comment_obj_id is None: + return None + if self._session is not None: + _proj = self._session.connect_project(self.comment_obj_id) + else: + _proj = project.Project(id=self.comment_obj_id) + return _proj.comment_by_id(self.comment_id) + elif self.comment_type == 1: + return user.User(username=self.comment_obj_title, _session=self._session).comment_by_id(self.comment_id) + elif self.comment_type == 2: + return user.User(id=self.comment_obj_id, _session=self._session).comment_by_id(self.comment_id) + else: + return None + + def target(self): + """ + Returns the activity's target (depending on the activity, this is either a User, Project, Studio or Comment object). + May also return None if the activity type is unknown. + """ + if self.type is None: + return None + _type = self.type.value + if self.type in ( + ActivityTypes.addprojecttostudio, + ActivityTypes.favoriteproject, + ActivityTypes.loveproject, + ActivityTypes.remixproject, + ActivityTypes.removeprojectfromstudio, + ActivityTypes.shareproject, + ): + return self.target_project() + if self.type in (ActivityTypes.becomecurator, ActivityTypes.followstudio): + if ret := self.target_studio(): + return ret + return self.target_user() + if self.type in (ActivityTypes.followuser, ActivityTypes.curatorinvite) or self.recipient_username: + return self.target_user() + if self.type == ActivityTypes.addcomment: + if ret := self.target_comment(): + return ret + raise ValueError(f"Either {self.comment_type} is an invalid comment type, or the linked target could not be found") + if _type == "forumpost": + return forum.ForumTopic(id=603418, _session=self._session, title=self.title) + return None diff --git a/scratchattach/sync_api/site/alert.py b/scratchattach/sync_api/site/alert.py new file mode 100644 index 00000000..1ae9e2fc --- /dev/null +++ b/scratchattach/sync_api/site/alert.py @@ -0,0 +1,177 @@ +from __future__ import annotations +import json +import pprint +import warnings +from dataclasses import dataclass, field, KW_ONLY +from datetime import datetime +from typing_extensions import TYPE_CHECKING, Any, Optional, Union, Self +from . import user, project, studio, comment, session +from scratchattach.utils import enums + +if TYPE_CHECKING: + ... + + +@dataclass +class EducatorAlert: + """ + Represents an alert for student activity, viewable at https://scratch.mit.edu/site-api/classrooms/alerts/ + + Attributes: + model: The type of alert (presumably); should always equal "educators.educatoralert" in this class + type: An integer that identifies the type of alert, differentiating e.g. against bans or autoban or censored comments etc + raw: The raw JSON data from the API + id: The ID of the alert (internally called 'pk' by scratch, not sure what this is for) + time_read: The time the alert was read + time_created: The time the alert was created + target: The user that the alert is about (the student) + actor: The user that created the alert (the admin) + target_object: The object that the alert is about (e.g. a project, studio, or comment) + notification_type: not sure what this is for, but inferred from the scratch HTML reference + """ + + _: KW_ONLY + target: user.User + actor: user.User + target_object: Optional[Union[project.Project, studio.Studio, comment.Comment, studio.Studio]] + notification_type: str + _session: Optional[session.Session] + model: str = "educators.educatoralert" + type: int = -1 + raw: dict = field(repr=False, default_factory=dict) + id: int = -1 + time_read: datetime = datetime.fromtimestamp(0.0) + time_created: datetime = datetime.fromtimestamp(0.0) + + @classmethod + def from_json(cls, data: dict[str, Any], _session: Optional[session.Session] = None) -> Self: + """ + Load an EducatorAlert from a JSON object. + + Arguments: + data (dict): The JSON object + _session (session.Session): The session object used to load this data, to 'connect' to the alerts rather than just 'get' them + + Returns: + EducatorAlert: The loaded EducatorAlert object + """ + model = data.get("model") + assert isinstance(model, str) + alert_id = data.get("pk") + assert isinstance(alert_id, int) + fields = data.get("fields") + assert isinstance(fields, dict) + time_read_raw = fields.get("educator_datetime_read") + assert isinstance(time_read_raw, str) + time_read: datetime = datetime.fromisoformat(time_read_raw) + admin_action = fields.get("admin_action") + assert isinstance(admin_action, dict) + time_created_raw = admin_action.get("datetime_created") + assert isinstance(time_created_raw, str) + time_created: datetime = datetime.fromisoformat(time_created_raw) + alert_type = admin_action.get("type") + assert isinstance(alert_type, int) + target_data = admin_action.get("target_user") + assert isinstance(target_data, dict) + target = user.User( + username=target_data.get("username"), + id=target_data.get("pk"), + icon_url=target_data.get("thumbnail_url"), + admin=target_data.get("admin", False), + _session=_session, + ) + actor_data = admin_action.get("actor") + assert isinstance(actor_data, dict) + actor = user.User( + username=actor_data.get("username"), + id=actor_data.get("pk"), + icon_url=actor_data.get("thumbnail_url"), + admin=actor_data.get("admin", False), + _session=_session, + ) + object_id = admin_action.get("object_id") + assert isinstance(object_id, int) + target_object: project.Project | studio.Studio | comment.Comment | None = None + extra_data: dict[str, Any] = json.loads(admin_action.get("extra_data", "{}")) + notification_type: str = "" + if "project_title" in extra_data: + target_object = project.Project(id=object_id, title=extra_data["project_title"], _session=_session) + elif "comment_content" in extra_data: + comment_data: dict[str, Any] = extra_data["comment_content"] + content: str | None = comment_data.get("content") + comment_obj_id: int | None = comment_data.get("comment_obj_id") + comment_type: int | None = comment_data.get("comment_type") + if comment_type == 0: + comment_source_type = comment.CommentSource.PROJECT + elif comment_type == 1: + comment_source_type = comment.CommentSource.USER_PROFILE + else: + comment_source_type = comment.CommentSource.STUDIO + warnings.warn( + f'The parser was not able to recognise the "comment_type" of {comment_type} in the alert JSON response.\nFull response: \n{pprint.pformat(data)}.\n\nPlease draft an issue on github: https://github.com/TimMcCool/scratchattach/issues, providing this whole error message. This will allow us to implement an incomplete part of this parser' + ) + target_object = comment.Comment( + id=object_id, content=content, source=comment_source_type, source_id=comment_obj_id, _session=_session + ) + elif "gallery_title" in extra_data: + target_object = studio.Studio(id=object_id, title=extra_data["gallery_title"], _session=_session) + elif "notification_type" in extra_data: + notification_type = extra_data["notification_type"] + else: + warnings.warn( + f'The parser was not able to recognise the "extra_data" in the alert JSON response.\nFull response: \n{pprint.pformat(data)}.\n\nPlease draft an issue on github: https://github.com/TimMcCool/scratchattach/issues, providing this whole error message. This will allow us to implement an incomplete part of this parser' + ) + return cls( + id=alert_id, + model=model, + type=alert_type, + raw=data, + time_read=time_read, + time_created=time_created, + target=target, + actor=actor, + target_object=target_object, + notification_type=notification_type, + _session=_session, + ) + + def __str__(self): + return f"EducatorAlert: {self.message}" + + @property + def alert_type(self) -> enums.AlertType: + """ + Get an associated AlertType object for this alert (based on the type index) + """ + alert_type = enums.AlertTypes.find(self.type) + if not alert_type: + alert_type = enums.AlertTypes.default.value + return alert_type + + @property + def message(self): + """ + Format the alert message using the alert type's message template, as it would be on the website. + """ + raw_message = self.alert_type.message + comment_content = "" + if isinstance(self.target_object, comment.Comment): + comment_content = self.target_object.content + return raw_message.format( + username=self.target.username, + project=self.target_object_title, + studio=self.target_object_title, + notification_type=self.notification_type, + comment=comment_content, + ) + + @property + def target_object_title(self): + """ + Get the title of the target object (if applicable) + """ + if isinstance(self.target_object, project.Project): + return self.target_object.title + if isinstance(self.target_object, studio.Studio): + return self.target_object.title + return None diff --git a/scratchattach/sync_api/site/backpack_asset.py b/scratchattach/sync_api/site/backpack_asset.py new file mode 100644 index 00000000..69da6c8e --- /dev/null +++ b/scratchattach/sync_api/site/backpack_asset.py @@ -0,0 +1,104 @@ +from __future__ import annotations +import json +import time +import logging +import warnings +from dataclasses import dataclass +from typing import Any, TYPE_CHECKING +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions +from scratchattach.utils.requests import requests + +if TYPE_CHECKING: + from scratchattach import session + + +@dataclass +class BackpackAsset(BaseSiteComponent): + """ + Represents an asset from the backpack. + + Attributes: + + :.id: + + :.type: The asset type (costume, script etc.) + + :.mime: The format in which the content of the backpack asset is saved + + :.name: The name of the backpack asset + + :.filename: Filename of the file containing the content of the backpack asset + + :.thumbnail_url: Link that leads to the asset's thumbnail (the image shown in the backpack UI) + + :.download_url: Link that leads to a file containing the content of the backpack asset + """ + + id: str + _session: session.Session | None = None + type: str | None = None + mime: str | None = None + name: str | None = None + filename: str | None = None + thumbnail_url: str | None = None + download_url: str | None = None + + def __repr__(self) -> str: + return f"BackpackAsset({self.filename})" + + def update(self): + warnings.warn("Warning: BackpackAsset objects can't be updated") + return False + + def _update_from_data(self, data: dict[str, str]) -> bool: + self.id = data.get("id", self.id) + self.type = data.get("type", self.type) + self.mime = data.get("mime", self.mime) + self.name = data.get("name", self.name) + self.filename = data.get("body", self.filename) + if "thumbnail" in data: + self.thumbnail_url = "https://backpack.scratch.mit.edu/" + data["thumbnail"] + if "body" in data: + self.download_url = "https://backpack.scratch.mit.edu/" + data["body"] + return True + + @property + def _data_bytes(self) -> bytes: + try: + with requests.no_error_handling(): + return requests.get(self.download_url).content + except Exception as e: + raise exceptions.FetchError(f"Failed to download asset: {e}") + + @property + def file_ext(self): + return self.filename.split(".")[-1] + + @property + def is_json(self): + return self.file_ext == "json" + + @property + def data(self) -> dict | list | int | None | str | bytes | float: + if self.is_json: + return json.loads(self._data_bytes) + else: + return self._data_bytes + + def download(self, *, fp: str = ""): + """ + Downloads the asset content to the given directory. The given filename is equal to the value saved in the .filename attribute. + + Args: + fp (str): The path of the directory the file will be saved in. + """ + if not (fp.endswith("/") or fp.endswith("\\")): + fp = fp + "/" + open(f"{fp}{self.filename}", "wb").write(self._data_bytes) + + def delete(self): + self._assert_auth() + return requests.delete( + f"https://backpack.scratch.mit.edu/{self._session.username}/{self.id}", headers=self._session._headers, timeout=10 + ).json() diff --git a/scratchattach/sync_api/site/browser_cookie3_stub.py b/scratchattach/sync_api/site/browser_cookie3_stub.py new file mode 100644 index 00000000..d535effe --- /dev/null +++ b/scratchattach/sync_api/site/browser_cookie3_stub.py @@ -0,0 +1,46 @@ +import http.cookiejar +from typing import Optional + + +def chrome(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def chromium(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def firefox(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def opera(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def edge(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def brave(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def vivaldi(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def safari(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def lynx(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def w3m(cookie_file: Optional[str] = None, key_file: Optional[str] = None) -> http.cookiejar.CookieJar: + return NotImplemented + + +def load() -> http.cookiejar.CookieJar: + return NotImplemented diff --git a/scratchattach/sync_api/site/browser_cookies.py b/scratchattach/sync_api/site/browser_cookies.py new file mode 100644 index 00000000..7f50ff84 --- /dev/null +++ b/scratchattach/sync_api/site/browser_cookies.py @@ -0,0 +1,64 @@ +from typing import Optional, TYPE_CHECKING +from typing_extensions import assert_never +from http.cookiejar import CookieJar +from enum import Enum, auto + +browsercookie_err = None +try: + if TYPE_CHECKING: + from . import browser_cookie3_stub as browser_cookie3 + else: + import browser_cookie3 +except Exception as e: + browsercookie = None + browsercookie_err = e + + +class Browser(Enum): + ANY = auto() + FIREFOX = auto() + CHROME = auto() + EDGE = auto() + SAFARI = auto() + CHROMIUM = auto() + VIVALDI = auto() + EDGE_DEV = auto() + + +FIREFOX = Browser.FIREFOX +CHROME = Browser.CHROME +EDGE = Browser.EDGE +SAFARI = Browser.SAFARI +CHROMIUM = Browser.CHROMIUM +VIVALDI = Browser.VIVALDI +ANY = Browser.ANY +EDGE_DEV = Browser.EDGE_DEV + + +def cookies_from_browser(browser: Browser = ANY) -> dict[str, str]: + """ + Import cookies from browser to login + """ + if not browser_cookie3: + raise browsercookie_err or ModuleNotFoundError() + cookies: Optional[CookieJar] = None + if browser is Browser.ANY: + cookies = browser_cookie3.load() + elif browser is Browser.FIREFOX: + cookies = browser_cookie3.firefox() + elif browser is Browser.CHROME: + cookies = browser_cookie3.chrome() + elif browser is Browser.EDGE: + cookies = browser_cookie3.edge() + elif browser is Browser.SAFARI: + cookies = browser_cookie3.safari() + elif browser is Browser.CHROMIUM: + cookies = browser_cookie3.chromium() + elif browser is Browser.VIVALDI: + cookies = browser_cookie3.vivaldi() + elif browser is Browser.EDGE_DEV: + raise ValueError("EDGE_DEV is not supported anymore.") + else: + assert_never(browser) + assert isinstance(cookies, CookieJar) + return {cookie.name: cookie.value for cookie in cookies if "scratch.mit.edu" in cookie.domain and cookie.value} diff --git a/scratchattach/sync_api/site/classroom.py b/scratchattach/sync_api/site/classroom.py new file mode 100644 index 00000000..4cda253c --- /dev/null +++ b/scratchattach/sync_api/site/classroom.py @@ -0,0 +1,417 @@ +from __future__ import annotations +import json +import warnings +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional, TYPE_CHECKING, Any, Callable +import bs4 +from bs4 import BeautifulSoup + +if TYPE_CHECKING: + from scratchattach.site.session import Session +from scratchattach.utils.commons import requests +from . import user, activity, typed_dicts +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions, commons +from scratchattach.utils.commons import headers + + +@dataclass +class Classroom(BaseSiteComponent): + title: str = "" + id: int = 0 + classtoken: str = "" + author: Optional[user.User] = None + about_class: str = "" + working_on: str = "" + is_closed: bool = False + datetime: datetime = datetime.fromtimestamp(0.0) + update_function: Callable = field(repr=False, default=requests.get) + _session: Optional[Session] = field(repr=False, default=None) + + def __post_init__(self): + if self.id: + self.update_api = f"https://api.scratch.mit.edu/classrooms/{self.id}" + elif self.classtoken: + self.update_api = f"https://api.scratch.mit.edu/classtoken/{self.classtoken}" + else: + raise KeyError(f"No class id or token provided! self.__dict__ = {self.__dict__!r}") + if self._session is None: + self._headers = commons.headers + self._cookies = {} + else: + self._headers = self._session._headers + self._cookies = self._session._cookies + self._json_headers = {**self._headers, "accept": "application/json", "Content-Type": "application/json"} + + def __str__(self) -> str: + return f"" + + def update(self): + try: + success = super().update() + except exceptions.ClassroomNotFound: + success = False + if not success: + response = requests.get(f"https://scratch.mit.edu/classes/{self.id}/") + soup = BeautifulSoup(response.text, "html.parser") + headings = soup.find_all("h1") + for heading in headings: + if heading.text == "Whoops! Our server is Scratch'ing its head": + raise exceptions.ClassroomNotFound(f"Classroom id {self.id} is not closed and cannot be found.") + title = soup.find("title").contents[0][: -len(" on Scratch")] + overviews = soup.find_all("p", {"class": "overview"}) + description, status = (overviews[0].text, overviews[1].text) + educator_username = None + pfx = "Scratch.INIT_DATA.PROFILE = {\n model: {\n id: '" + sfx = "',\n userId: " + for script in soup.find_all("script"): + if pfx in script.text: + educator_username = commons.webscrape_count(script.text, pfx, sfx, str) + ret: typed_dicts.ClassroomDict = { + "id": self.id, + "title": title, + "description": description, + "educator": {}, + "status": status, + "is_closed": True, + } + if educator_username: + ret["educator"]["username"] = educator_username + return self._update_from_data(ret) + return success + + def _update_from_data(self, data: typed_dicts.ClassroomDict): + self.id = int(data["id"]) + self.title = data["title"] + self.about_class = data["description"] + self.working_on = data["status"] + self.datetime = datetime.fromisoformat(data["date_start"]) + self.author = user.User(username=data["educator"]["username"], _session=self._session) + self.author.supply_data_dict(data["educator"]) + self.is_closed = bool(data["date_end"]) + return True + + def student_count(self) -> int: + text = requests.get(f"https://scratch.mit.edu/classes/{self.id}/", headers=self._headers).text + return commons.webscrape_count(text, "Students (", ")") + + def student_names(self, *, page=1) -> list[str]: + """ + Returns the student on the class. + + Keyword Arguments: + page: The page of the students that should be returned. + + Returns: + list: The usernames of the class students + """ + if self.is_closed: + ret = [] + response = requests.get(f"https://scratch.mit.edu/classes/{self.id}/") + soup = BeautifulSoup(response.text, "html.parser") + found = set("") + for result in soup.css.select("ul.scroll-content .user a"): + result_text = result.text.strip() + if result_text in found: + continue + found.add(result_text) + ret.append(result_text) + return ret + text = requests.get(f"https://scratch.mit.edu/classes/{self.id}/students/?page={page}", headers=self._headers).text + textlist = [i.split('/">')[0] for i in text.split(' list[int]: + """ + Returns the class studio on the class. + + Keyword Arguments: + page: The page of the students that should be returned. + + Returns: + list: The id of the class studios + """ + if self.is_closed: + ret = [] + response = requests.get(f"https://scratch.mit.edu/classes/{self.id}/") + soup = BeautifulSoup(response.text, "html.parser") + for result in soup.css.select("ul.scroll-content .gallery a[href]:not([class])"): + value = result["href"] + if not isinstance(value, str): + value = value[0] + ret.append(commons.webscrape_count(value, "/studios/", "/")) + return ret + text = requests.get(f"https://scratch.mit.edu/classes/{self.id}/studios/?page={page}", headers=self._headers).text + textlist = [int(i.split('/">')[0]) for i in text.split('\n None: + self._check_session() + requests.post( + f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, + cookies=self._cookies, + files={"file": thumbnail}, + ) + + def set_description(self, desc: str) -> None: + self._check_session() + response = requests.put( + f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, + cookies=self._cookies, + json={"description": desc}, + ) + try: + data = response.json() + if data["description"] == desc: + return + else: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def set_working_on(self, status: str) -> None: + self._check_session() + response = requests.put( + f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, + cookies=self._cookies, + json={"status": status}, + ) + try: + data = response.json() + if data["status"] == status: + return + else: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def set_title(self, title: str) -> None: + self._check_session() + response = requests.put( + f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, + cookies=self._cookies, + json={"title": title}, + ) + try: + data = response.json() + if data["title"] == title: + return + else: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def add_studio(self, name: str, description: str = "") -> None: + self._check_session() + requests.post( + "https://scratch.mit.edu/classes/create_classroom_gallery/", + json={"classroom_id": str(self.id), "classroom_token": self.classtoken, "title": name, "description": description}, + headers=self._headers, + cookies=self._cookies, + ) + + def reopen(self) -> None: + self._check_session() + response = requests.put( + f"https://scratch.mit.edu/site-api/classrooms/all/{self.id}/", + headers=self._headers, + cookies=self._cookies, + json={"visibility": "visible"}, + ) + try: + response.json() + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def close(self) -> None: + self._check_session() + response = requests.post( + f"https://scratch.mit.edu/site-api/classrooms/close_classroom/{self.id}/", + headers=self._headers, + cookies=self._cookies, + ) + try: + response.json() + except Exception as e: + warnings.warn(f"{self._session} may not be authenticated to edit {self}") + raise e + + def register_student( + self, + username: str, + password: str = "", + birth_month: Optional[int] = None, + birth_year: Optional[int] = None, + gender: Optional[str] = None, + country: Optional[str] = None, + is_robot: bool = False, + ) -> None: + return register_by_token( + self.id, + self.classtoken, + username, + password, + birth_month or 1, + birth_year or 2000, + gender or "(Prefer not to say)", + country or "United+States", + is_robot, + ) + + def generate_signup_link(self): + if self.classtoken is not None: + return f"https://scratch.mit.edu/signup/{self.classtoken}" + self._check_session() + response = requests.get( + f"https://scratch.mit.edu/site-api/classrooms/generate_registration_link/{self.id}/", + headers=self._headers, + cookies=self._cookies, + ) + data = response.json() + if "reg_link" in data: + return data["reg_link"] + else: + raise exceptions.Unauthorized(f"{self._session} is not authorised to generate a signup link of {self}") + + def public_activity(self, *, limit=20): + """ + Returns: + list: The user's activity data as parsed list of scratchattach.activity.Activity objects + """ + if limit > 20: + warnings.warn("The limit is set to more than 20. There may be an error") + soup = BeautifulSoup( + requests.get(f"https://scratch.mit.edu/site-api/classrooms/activity/public/{self.id}/?limit={limit}").text, + "html.parser", + ) + activities = [] + source = soup.find_all("li") + for data in source: + _activity = activity.Activity(_session=self._session, raw=data) + _activity._update_from_html(data) + activities.append(_activity) + return activities + + def activity( + self, student: str = "all", mode: str = "Last created", page: Optional[int] = None + ) -> list[activity.Activity]: + """ + Get a list of private activity, only available to the class owner. + Returns: + list The private activity of users in the class + """ + self._check_session() + ascsort, descsort = commons.get_class_sort_mode(mode) + with requests.no_error_handling(): + try: + data = requests.get( + f"https://scratch.mit.edu/site-api/classrooms/activity/{self.id}/{student}/", + params={"page": page, "ascsort": ascsort, "descsort": descsort}, + headers=self._headers, + cookies=self._cookies, + ).json() + except json.JSONDecodeError: + return [] + _activity: list[activity.Activity] = [] + for activity_json in data: + _activity.append(activity.Activity(_session=self._session)) + _activity[-1]._update_from_json(activity_json) + return _activity + + +def get_classroom(class_id: str) -> Classroom: + """ + Gets a class without logging in. + + Args: + class_id (str): class id of the requested class + + Returns: + scratchattach.classroom.Classroom: An object representing the requested classroom + + Warning: + Any methods that require authentication will not work on the returned object. + + If you want to use these, get the user with :meth:`scratchattach.session.Session.connect_classroom` instead. + """ + warnings.warn( + "For methods that require authentication, use session.connect_classroom instead of get_classroom\nIf you want to remove this warning, use warnings.filterwarnings('ignore', category=scratchattach.ClassroomAuthenticationWarning)\nTo ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use `warnings.filterwarnings('ignore', category=scratchattach.GetAuthenticationWarning)`.", + exceptions.ClassroomAuthenticationWarning, + ) + return commons._get_object("id", class_id, Classroom, exceptions.ClassroomNotFound) + + +def get_classroom_from_token(class_token) -> Classroom: + """ + Gets a class without logging in. + + Args: + class_token (str): class token of the requested class + + Returns: + scratchattach.classroom.Classroom: An object representing the requested classroom + + Warning: + Any methods that require authentication will not work on the returned object. + + If you want to use these, get the user with :meth:`scratchattach.session.Session.connect_classroom` instead. + """ + warnings.warn( + "For methods that require authentication, use session.connect_classroom instead of get_classroom. If you want to remove this warning, use warnings.filterwarnings('ignore', category=ClassroomAuthenticationWarning). To ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use warnings.filterwarnings('ignore', category=GetAuthenticationWarning).", + exceptions.ClassroomAuthenticationWarning, + ) + return commons._get_object("classtoken", class_token, Classroom, exceptions.ClassroomNotFound) + + +def register_by_token( + class_id: int, + class_token: str, + username: str, + password: str, + birth_month: int, + birth_year: int, + gender: str, + country: str, + is_robot: bool = False, +) -> None: + data = { + "classroom_id": class_id, + "classroom_token": class_token, + "username": username, + "password": password, + "birth_month": birth_month, + "birth_year": birth_year, + "gender": gender, + "country": country, + "is_robot": is_robot, + } + response = requests.post( + "https://scratch.mit.edu/classes/register_new_student/", + data=data, + headers=commons.headers, + cookies={"scratchcsrftoken": "a"}, + ) + ret = response.json()[0] + if "username" in ret: + return + else: + raise exceptions.Unauthorized(f"Can't create account: {response.text}") diff --git a/scratchattach/sync_api/site/cloud_activity.py b/scratchattach/sync_api/site/cloud_activity.py new file mode 100644 index 00000000..39b31fad --- /dev/null +++ b/scratchattach/sync_api/site/cloud_activity.py @@ -0,0 +1,122 @@ +from __future__ import annotations +import time +from typing import Union, TypeGuard, Optional +from dataclasses import dataclass, field +import warnings +from scratchattach.cloud import _base +from scratchattach.utils import exceptions +from scratchattach.site import user +from scratchattach.site import project as project_module +from ._base import BaseSiteComponent +from . import typed_dicts, session + + +@dataclass +class CloudActivity(BaseSiteComponent[Union[typed_dicts.CloudActivityDict, typed_dicts.CloudLogActivityDict]]): + """ + Represents a cloud activity (a cloud variable set / creation / deletion). + """ + + username: str = field(kw_only=True, default="") + "The user who caused the cloud event (the user who added / set / deleted the cloud variable)" + var: str = field(kw_only=True, default="") + "The name of the cloud variable that was updated (specified without the cloud emoji)" + name: str = field(kw_only=True, default="") + "The name of the cloud variable that was updated (specified without the cloud emoji)" + actual_var: str = field(kw_only=True, default="") + type: str = field(kw_only=True, default="set") + "The activity type" + timestamp: float = field(kw_only=True, default=0.0) + "Then timestamp of when the action was performed" + value: Union[float, int, str] = field(kw_only=True, default="0.0") + "If the cloud variable was set, then this attribute provides the value the cloud variable was set to" + cloud: _base.AnyCloud = field(kw_only=True, default_factory=lambda: _base.DummyCloud()) + "The cloud (as object inheriting from scratchattach.Cloud.BaseCloud) that the cloud activity corresponds to" + _session: Optional[session.Session] = field(kw_only=True, default=None) + + def update(self): + warnings.warn("CloudActivity objects can't be updated", exceptions.InvalidUpdateWarning) + return False + + def __eq__(self, activity2): + return ( + self.user == activity2.user + and self.type == activity2.type + and (self.timestamp == activity2.timestamp) + and (self.value == activity2.value) + and (self.name == activity2.name) + ) + + def _update_from_data(self, data: Union[typed_dicts.CloudActivityDict, typed_dicts.CloudLogActivityDict]) -> bool: + + def is_cloud_log_activity( + activity: Union[typed_dicts.CloudActivityDict, typed_dicts.CloudLogActivityDict], + ) -> TypeGuard[typed_dicts.CloudLogActivityDict]: + return "verb" in activity + + def is_cloud_activity( + activity: Union[typed_dicts.CloudActivityDict, typed_dicts.CloudLogActivityDict], + ) -> TypeGuard[typed_dicts.CloudActivityDict]: + return "method" in activity + + self.name = data["name"] + self.var = data["name"] + self.value = data["value"] + self.actual_var = data.get("variable_name") or self.var + if is_cloud_log_activity(data): + self.user = data["user"] + self.username = data["user"] + self.timestamp = data["timestamp"] + self.type = data["verb"].removesuffix("_var") + elif is_cloud_activity(data): + self.type = data["method"] + if "cloud" in data: + self.cloud = data["cloud"] + return True + + def a(self, **k): + pass + + def load_log_data(self): + if self.cloud is None: + print( + "Warning: There aren't cloud logs available for this cloud, therefore the user and exact timestamp can't be loaded" + ) + elif isinstance(self.cloud, _base.LogCloud): + logs = self.cloud.logs(filter_by_var_named=self.var, limit=100) + matching = list(filter(lambda x: x.value == self.value and x.timestamp <= self.timestamp, logs)) + if matching == []: + return False + activity = matching[0] + self.username = activity.username + self.user = activity.username + self.timestamp = activity.timestamp + return True + else: + print( + "Warning: There aren't cloud logs available for this cloud, therefore the user and exact timestamp can't be loaded" + ) + return False + + def actor(self): + """ + Returns the user that performed the cloud activity as scratchattach.user.User object + """ + if self.username is None: + return None + return self._make_linked_object("username", self.username, user.User, exceptions.UserNotFound) + + def project(self) -> Optional[project_module.Project]: + """ + Returns the project where the cloud activity was performed as scratchattach.project.Project object + """ + + def make_linked(cloud: _base.BaseCloud) -> project_module.Project: + return self._make_linked_object("id", cloud.project_id, project_module.Project, exceptions.ProjectNotFound) + + if self.cloud is None: + return None + cloud = self.cloud + if not isinstance(cloud, _base.BaseCloud): + return None + return make_linked(cloud) diff --git a/scratchattach/sync_api/site/comment.py b/scratchattach/sync_api/site/comment.py new file mode 100644 index 00000000..ab735c17 --- /dev/null +++ b/scratchattach/sync_api/site/comment.py @@ -0,0 +1,207 @@ +"""Comment class""" + +from __future__ import annotations +import warnings +import html +from dataclasses import dataclass +from typing_extensions import assert_never +from typing import Union, Optional, Any +from enum import Enum, auto +from . import user, project, studio, session +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions + + +class CommentSource(Enum): + PROJECT = auto() + USER_PROFILE = auto() + STUDIO = auto() + UNKNOWN = auto() + + +@dataclass +class Comment(BaseSiteComponent): + """ + Represents a Scratch comment (on a profile, studio or project) + """ + + id: Optional[int | str] = None + source: CommentSource = CommentSource.UNKNOWN + source_id: Optional[int | str] = None + cached_replies: Optional[list[Comment]] = None + parent_id: Optional[int | str] = None + cached_parent_comment: Optional[Comment] = None + commentee_id: Optional[int] = None + content: Optional[str] = None + reply_count: Optional[int] = None + written_by_scratchteam: Optional[bool] = None + author_id: Optional[int] = None + author_name: Optional[str] = None + _session: Optional[session.Session] = None + + def __str__(self): + return self.text + + def update(self): + warnings.warn("Warning: Comment objects can't be updated") + return False + + def _update_from_data(self, data: dict[str, str | dict | Any]): + self.id = data["id"] + self.parent_id = data.get("parent_id") + self.commentee_id = data.get("commentee_id") + self.content = str(data["content"]) + self.datetime_created = data["datetime_created"] + author = data.get("author", {}) + self.author_name = author.get("username", self.author_name) + self.author_id = author.get("id", self.author_id) + self.written_by_scratchteam = author.get("scratchteam", self.written_by_scratchteam) + self.reply_count = data.get("reply_count", self.reply_count) + source: str = data.get("source") + if self.source is CommentSource.UNKNOWN: + self.source = { + "project": CommentSource.PROJECT, + "studio": CommentSource.STUDIO, + "profile": CommentSource.USER_PROFILE, + None: CommentSource.UNKNOWN, + }[source] + self.source_id = data.get("source_id", self.source_id) + + @property + def text(self) -> str: + """ + Parsed version of Comment.content. This removes any escape codes, e.g. ''' becomes ', an apostrophe + """ + if self.source is CommentSource.USER_PROFILE: + return self.content + return str(html.unescape(self.content)) + + def author(self) -> user.User: + return self._make_linked_object("username", self.author_name, user.User, exceptions.UserNotFound) + + def place(self) -> user.User | studio.Studio | project.Project: + """ + Returns the place (the project, profile or studio) where the comment was posted as Project, User, or Studio object. + + If the place can't be traced back, None is returned. + """ + if self.source == CommentSource.USER_PROFILE: + return self._make_linked_object("username", self.source_id, user.User, exceptions.UserNotFound) + elif self.source == CommentSource.STUDIO: + return self._make_linked_object("id", self.source_id, studio.Studio, exceptions.UserNotFound) + elif self.source == CommentSource.PROJECT: + return self._make_linked_object("id", self.source_id, project.Project, exceptions.UserNotFound) + else: + assert_never(self.source) + + def parent_comment(self) -> Comment | None: + if self.parent_id is None: + return None + if self.cached_parent_comment is not None: + return self.cached_parent_comment + if self.source == "profile": + self.cached_parent_comment = user.User(username=self.source_id, _session=self._session).comment_by_id( + self.parent_id + ) + elif self.source == "project": + p = project.Project(id=self.source_id, _session=self._session) + p.update() + self.cached_parent_comment = p.comment_by_id(self.parent_id) + elif self.source == "studio": + self.cached_parent_comment = studio.Studio(id=self.source_id, _session=self._session).comment_by_id(self.parent_id) + return self.cached_parent_comment + + def replies(self, *, use_cache: bool = True, limit=40, offset=0): + """ + Keyword Arguments: + use_cache (bool): Returns the replies cached on the first reply fetch. This makes it SIGNIFICANTLY faster for profile comments. Warning: For profile comments, the replies are retrieved and cached on object creation. + """ + if self.cached_replies is None or not use_cache: + if self.source == CommentSource.USER_PROFILE: + self.cached_replies = ( + user.User(username=self.source_id, _session=self._session) + .comment_by_id(self.id) + .cached_replies[offset : offset + limit] + ) + elif self.source == CommentSource.PROJECT: + p = project.Project(id=self.source_id, _session=self._session) + p.update() + self.cached_replies = p.comment_replies(comment_id=self.id, limit=limit, offset=offset) + elif self.source == CommentSource.STUDIO: + self.cached_replies = studio.Studio(id=self.source_id, _session=self._session).comment_replies( + comment_id=self.id, limit=limit, offset=offset + ) + return self.cached_replies + + def reply(self, content, *, commentee_id=None): + """ + Posts a reply comment to the comment. + + Warning: + Scratch only shows comments replying to top-level comments, and all replies to replies are actually replies to top-level comments in the API. + + Therefore, if this comment is a reply, this method will not reply to the comment itself but to the corresponding top-level comment. + + Args: + content (str): Comment content to post. + + Keyword args: + commentee_id (None or str): If set to None (default), it will automatically fill out the commentee ID with the user ID of the parent comment author. Set it to "" to mention no user. + + + Returns: + scratchattach.Comment: The created comment. + :param content: Content of the comment to send + :param commentee_id: ID of user to reply to + """ + self._assert_auth() + parent_id = str(self.id) + if self.parent_id is not None: + parent_id = str(self.parent_id) + if commentee_id is None: + if self.author_id: + commentee_id = self.author_id + else: + commentee_id = "" + if self.source == CommentSource.USER_PROFILE: + return user.User(username=self.source_id, _session=self._session).reply_comment( + content, parent_id=str(parent_id), commentee_id=commentee_id + ) + if self.source == CommentSource.PROJECT: + p = project.Project(id=self.source_id, _session=self._session) + p.update() + return p.reply_comment(content, parent_id=str(parent_id), commentee_id=commentee_id) + if self.source == CommentSource.STUDIO: + return studio.Studio(id=self.source_id, _session=self._session).reply_comment( + content, parent_id=str(parent_id), commentee_id=commentee_id + ) + raise ValueError(f"Unknown source: {self.source}") + + def delete(self): + """ + Deletes the comment. + """ + self._assert_auth() + if self.source == CommentSource.USER_PROFILE: + return user.User(username=self.source_id, _session=self._session).delete_comment(comment_id=self.id) + elif self.source == CommentSource.PROJECT: + p = project.Project(id=self.source_id, _session=self._session) + p.update() + return p.delete_comment(comment_id=self.id) + elif self.source == CommentSource.STUDIO: + return studio.Studio(id=self.source_id, _session=self._session).delete_comment(comment_id=self.id) + return None + + def report(self): + """ + Reports the comment to the Scratch team. + """ + self._assert_auth() + if self.source == CommentSource.USER_PROFILE: + user.User(username=self.source_id, _session=self._session).report_comment(comment_id=self.id) + elif self.source == CommentSource.PROJECT: + p = project.Project(id=self.source_id, _session=self._session) + p.update() + p.report_comment(comment_id=self.id) + elif self.source == CommentSource.STUDIO: + studio.Studio(id=self.source_id, _session=self._session).report_comment(comment_id=self.id) diff --git a/scratchattach/sync_api/site/forum.py b/scratchattach/sync_api/site/forum.py new file mode 100644 index 00000000..61bb25d1 --- /dev/null +++ b/scratchattach/sync_api/site/forum.py @@ -0,0 +1,416 @@ +"""ForumTopic and ForumPost classes""" + +from __future__ import annotations +import warnings +from dataclasses import dataclass, field +from typing import Optional, Any +from urllib.parse import urlparse, parse_qs +import xml.etree.ElementTree as ET +from bs4 import BeautifulSoup, Tag +from . import user +from . import session as module_session +from scratchattach.utils.commons import headers +from scratchattach.utils import exceptions, commons +from ._base import BaseSiteComponent +from scratchattach.utils.requests import requests + + +@dataclass +class ForumTopic(BaseSiteComponent): + """ + Represents a Scratch forum topic. + + Attributes: + + :.id: + + :.title: + + :.category_name: + + :.last_updated: + + Attributes only available if the object was created using scratchattach.get_topic_list or scratchattach.Session.connect_topic_list: + + :.reply_count: + + :.view_count: + + :.update(): Updates the attributes + """ + + id: int + title: str + category_name: Optional[str] = None + last_updated: Optional[str] = None + _session: Optional[module_session.Session] = field(default=None) + reply_count: Optional[int] = field(default=None) + view_count: Optional[int] = field(default=None) + + def __str__(self): + return f"-F {self.title} ({self.id})" + + def __post_init__(self): + self.update_function = requests.get + self.update_api = f"https://scratch.mit.edu/discuss/feeds/topic/{self.id}/" + if self._session is None: + self._headers = headers + self._cookies = {} + else: + self._headers = self._session.get_headers() + self._cookies = self._session.get_cookies() + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def update(self): + response = self.update_function(self.update_api, headers=self._headers, cookies=self._cookies, timeout=20) + if "429" in str(response): + return "429" + if response.status_code == 200: + try: + root = ET.fromstring(response.text) + namespace = {"atom": "http://www.w3.org/2005/Atom"} + title = root.findtext("atom:title", namespaces=namespace).replace("Latest posts on ", "") + category_name = ( + root.findall(".//atom:entry", namespaces=namespace)[0] + .findtext(".//atom:title", namespaces=namespace) + .split(" :: ")[1] + ) + last_updated = root.findtext("atom:updated", namespaces=namespace) + except Exception as e: + raise exceptions.ScrapeError(str(e)) + else: + raise exceptions.ForumContentNotFound + self.title = title + self.category_name = category_name + self.last_updated = last_updated + return True + + @classmethod + def from_id(cls, __id: int, session: module_session.Session, update: bool = False): + new = cls(id=__id, _session=session, title="", last_updated="", category_name="") + if update: + new.update() + return new + + def _update_from_data(self, data: dict[str, Any]): + self.__dict__.update(data) + + def posts(self, *, page=1, order="oldest") -> list[ForumPost]: + """ + Args: + page (int): The page of the forum topic that should be returned. First page is at index 1. + + Returns: + list: A list containing the posts from the specified page of the forum topic + """ + if order != "oldest": + warnings.warn("Warning: All post orders except for 'oldest' are deprecated and no longer work") + posts = [] + try: + url = f"https://scratch.mit.edu/discuss/topic/{self.id}/?page={page}" + response = requests.get(url, headers=headers, cookies=self._cookies) + except Exception as e: + raise exceptions.FetchError(str(e)) + try: + soup = BeautifulSoup(response.content, "html.parser") + soup_elm = soup.find("div", class_="djangobb") + assert isinstance(soup_elm, Tag) + try: + pagination_div = soup_elm.find("div", class_="pagination") + assert isinstance(pagination_div, Tag) + num_pages = int(pagination_div.find_all("a", class_="page")[-1].text) + except Exception: + num_pages = 1 + try: + topic_category = "" + breadcrumb_ul = soup_elm.find_all("ul")[1] + if breadcrumb_ul: + assert isinstance(breadcrumb_ul, Tag) + link = breadcrumb_ul.find_all("a")[1] + topic_category = link.text.strip() + except Exception as e: + warnings.warn(f"Warning: Couldn't scrape topic category for topic {self.id} - {e}") + topic_category = "" + post_htmls = soup.find_all("div", class_="blockpost") + for raw_post in post_htmls: + if not isinstance(raw_post, Tag): + continue + post = ForumPost( + id=int(str(raw_post["id"]).replace("p", "")), + topic_id=self.id, + _session=self._session, + topic_category=topic_category, + topic_num_pages=num_pages, + ) + post.update_from_html(raw_post) + posts.append(post) + except Exception as e: + raise exceptions.ScrapeError() from e + return posts + + def first_post(self): + """ + Returns: + scratchattach.forum.ForumPost: An object representing the first topic post + """ + posts = self.posts(page=1) + if len(posts) > 0: + return posts[0] + + +@dataclass +class ForumPost(BaseSiteComponent): + """ + Represents a Scratch forum post. + + Attributes: + + :.id: + + :.author_name: The name of the user who created this post + + :.author_avatar_url: + + :.posted: The date the post was made + + :.topic_id: The id of the topic this post is in + + :.topic_name: The name of the topic the post is in + + :.topic_category: The name of the category the post topic is in + + :.topic_num_pages: The number of pages the post topic has + + :.deleted: Whether the post was deleted (always False because deleted posts can't be retrieved anymore) + + :.html_content: Returns the content as HTML + + :.content: Returns the content as text + + :.post_index: The index that the post has in the topic + + :.update(): Updates the attributes + """ + + id: int = field(default=0) + topic_id: int = field(default=0) + topic_name: str = field(default="") + topic_category: str = field(default="") + topic_num_pages: int = field(default=0) + author_name: str = field(default="") + author_avatar_url: str = field(default="") + posted: str = field(default="") + deleted: bool = field(default=False) + html_content: str = field(default="") + content: str = field(default="") + post_index: int = field(default=0) + _session: Optional[module_session.Session] = field(default=None) + + def __post_init__(self): + self.update_api = "" + if self._session is None: + self._headers = headers + self._cookies = {} + else: + self._headers = self._session.get_headers() + self._cookies = self._session.get_cookies() + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def update_function(self, *args, **kwargs): + raise TypeError("Forum posts cannot be updated like this") + + def update(self): + """ + Updates the attributes of the ForumPost object. + As there is no API for retrieving a single post anymore, this requires reloading the forum page. + """ + page = 1 + posts = ForumTopic.from_id(self.topic_id, session=self._session).posts(page=1) + while posts != []: + matching = list(filter(lambda x: int(x.id) == int(self.id), posts)) + if len(matching) > 0: + this = matching[0] + break + page += 1 + posts = ForumTopic.from_id(self.topic_id, session=self._session).posts(page=page) + else: + return False + self._update_from_data(vars(this)) + + def _update_from_data(self, data: dict[str, Any]): + self.__dict__.update(data) + return True + + def update_from_html(self, soup_html: Tag): + return self._update_from_html(soup_html) + + def _update_from_html(self, soup_html: Tag): + post_index_elm = soup_html.find("span", class_="conr") + assert isinstance(post_index_elm, Tag) + id_attr = soup_html["id"] + assert isinstance(id_attr, str) + posted_elm = soup_html.find("a", href=True) + assert isinstance(posted_elm, Tag) + content_elm = soup_html.find("div", class_="post_body_html") + assert isinstance(content_elm, Tag) + author_name_elm = soup_html.select_one("dl dt a") + assert isinstance(author_name_elm, Tag) + topic_name_elm = soup_html.find("h3") + assert isinstance(topic_name_elm, Tag) + self.post_index = int(post_index_elm.text.strip("#")) + self.id = int(id_attr.replace("p", "")) + self.posted = posted_elm.text.strip() + self.content = content_elm.text.strip() + self.html_content = str(soup_html.find("div", class_="post_body_html")) + self.author_name = author_name_elm.text.strip() + self.author_avatar_url = str(author_name_elm["href"]) + self.topic_name = topic_name_elm.text.strip() + return True + + def topic(self): + """ + Returns: + scratchattach.forum.ForumTopic: An object representing the forum topic this post is in. + """ + return self._make_linked_object("id", self.topic_id, ForumTopic, exceptions.ForumContentNotFound) + + def ocular_reactions(self): + return requests.get(f"https://my-ocular.jeffalo.net/api/reactions/{self.id}", timeout=10).json() + + def author(self): + """ + Returns: + scratchattach.user.User: An object representing the user who created this forum post. + """ + return self._make_linked_object("username", self.author_name, user.User, exceptions.UserNotFound) + + def edit(self, new_content: str): + """ + Changes the content of the forum post. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_post` or through another method that requires authentication. You must own the forum post. + + Args: + new_content (str): The text that the forum post will be set to. + """ + self._assert_auth() + cookies = dict(self._cookies) + cookies["accept"] = ( + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" + ) + cookies["Content-Type"] = "application/x-www-form-urlencoded" + r = requests.post( + f"https://scratch.mit.edu/discuss/post/{self.id}/edit/", + headers={ + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "accept-language": "de,en;q=0.9", + "cache-control": "max-age=0", + "content-type": "application/x-www-form-urlencoded", + "sec-ch-ua": '" Not A;Brand";v="99", "Chromium";v="101", "Google Chrome";v="101"', + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": '"Windows"', + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "same-origin", + "sec-fetch-user": "?1", + "upgrade-insecure-requests": "1", + "Referer": f"https://scratch.mit.edu/discuss/post/{self.id}/edit/", + "x-csrftoken": "a", + }, + cookies=cookies, + json=f"csrfmiddlewaretoken=a&body={new_content}&", + timeout=10, + ) + + +def get_topic(topic_id) -> ForumTopic: + """ + Gets a forum topic without logging in. Data received from Scratch's RSS feed XML API. + + Args: + topic_id (int): ID of the requested forum topic + + Returns: + scratchattach.forum.ForumTopic: An object representing the requested forum topic + + Warning: + Scratch's API uses very heavy caching for logged out users, therefore the returned data will not be up to date. + + Any methods that require authentication will not work on the returned object. + + If you need up-to-date data or want to use methods that require authentication, create the object with :meth:`scratchattach.session.Session.connect_topic` instead. + """ + return commons._get_object("id", topic_id, ForumTopic, exceptions.ForumContentNotFound) + + +def get_topic_list(category_id, *, page=1): + """ + Gets the topics from a forum category without logging in. Data web-scraped from Scratch's forums UI. + + Args: + category_id (str): ID of the forum category + + Keyword Arguments: + page (str): Page of the category topics that should be returned + + Returns: + list: A list containing the forum topics from the specified category + + Warning: + Scratch's API uses very heavy caching for logged out users, therefore the returned data will not be up to date. + + Any methods that require authentication will not work on the returned objects. + + If you need up-to-date data or want to use methods that require authentication, get the forum topics with :meth:`scratchattach.session.Session.connect_topic_list` instead. + """ + try: + response = requests.get(f"https://scratch.mit.edu/discuss/{category_id}/?page={page}") + soup = BeautifulSoup(response.content, "html.parser") + except Exception as e: + raise exceptions.FetchError(str(e)) + try: + category_name = soup.find("h4").find("span").get_text() + except Exception as e: + raise exceptions.BadRequest("Invalid category id") + try: + topics = soup.find_all("tr") + topics.pop(0) + return_topics = [] + for topic in topics: + title_link = topic.find("a") + title = title_link.text.strip() + topic_id = title_link["href"].split("/")[-2] + columns = topic.find_all("td") + columns = [column.text for column in columns] + if len(columns) == 1: + continue + last_updated = columns[3].split(" ")[0] + " " + columns[3].split(" ")[1] + return_topics.append( + ForumTopic( + id=int(topic_id), + title=title, + category_name=category_name, + last_updated=last_updated, + reply_count=int(columns[1]), + view_count=int(columns[2]), + ) + ) + return return_topics + except Exception as e: + raise exceptions.ScrapeError(str(e)) + + +def youtube_link_to_scratch(link: str): + """ + Converts a YouTube url (in multiple formats) like https://youtu.be/1JTgg4WVAX8?si=fIEskaEaOIRZyTAz + to a link like https://scratch.mit.edu/discuss/youtube/1JTgg4WVAX8 + """ + url_parse = urlparse(link) + query_parse = parse_qs(url_parse.query) + if "v" in query_parse: + video_id = query_parse["v"][0] + else: + video_id = url_parse.path.split("/")[-1] + return f"https://scratch.mit.edu/discuss/youtube/{video_id}" diff --git a/scratchattach/sync_api/site/project.py b/scratchattach/sync_api/site/project.py new file mode 100644 index 00000000..a2ca7f2d --- /dev/null +++ b/scratchattach/sync_api/site/project.py @@ -0,0 +1,791 @@ +"""Project and PartialProject classes""" + +from __future__ import annotations +import json +import random +import base64 +import time +import warnings +import zipfile +from io import BytesIO +from typing import Callable, Union +from dataclasses import dataclass, field +from typing import Any, Optional +from typing_extensions import deprecated +from scratchattach.site.typed_dicts import ProjectDict +from . import user, comment, studio, session +from scratchattach.utils import exceptions +from scratchattach.utils import commons +from scratchattach.utils.commons import empty_project_json, headers +from ._base import BaseSiteComponent +from scratchattach import editor +from scratchattach.utils.requests import requests + +CREATE_PROJECT_USES: list[float] = [] + + +@dataclass +class PartialProject(BaseSiteComponent): + """ + Represents an unshared Scratch project that can't be accessed. + """ + + id: Union[str, int] = field(kw_only=True, default=0) + "The project id" + url: str = field(kw_only=True, default="") + "The project url" + title: str = field(kw_only=True, default="") + author_name: str = field(kw_only=True, default="") + "The username of the author" + comments_allowed: bool = field(kw_only=True, default=False) + "whether comments are enabled" + comment_count: int = 0 + "The number of comments on the project (this may be unreliable)" + instructions: str = field(kw_only=True, default="") + notes: str = field(kw_only=True, default="") + "The 'Notes and Credits' section" + created: str = field(kw_only=True, default="") + "The date of the project creation" + last_modified: str = field(kw_only=True, default="") + "The date when the project was modified the last time" + share_date: str = field(kw_only=True, default="") + thumbnail_url: str = field(kw_only=True, default="") + remix_parent: Optional[Union[str, int]] = field(kw_only=True, default="") + parent_title: Optional[str] = field(kw_only=True, default=None) + remix_root: Optional[Union[str, int]] = field(kw_only=True, default="") + loves: int = field(kw_only=True, default=0) + "The project's love count" + favorites: int = field(kw_only=True, default=0) + "The project's favorite count" + remix_count: int = field(kw_only=True, default=0) + "The number of remixes" + views: int = field(kw_only=True, default=0) + "The view count" + project_token: Optional[str] = field(kw_only=True, default=None) + "The project token (required to access the project json)" + _moderation_status: Optional[str] = field(kw_only=True, default=None) + _session: Optional[session.Session] = field(kw_only=True, default=None) + + def __str__(self): + return f"Unshared project with id {self.id}" + + def __post_init__(self) -> None: + self.update_function: Callable = requests.get + self.update_api = f"https://api.scratch.mit.edu/projects/{self.id}" + if self._session is None: + self._headers = headers + self._cookies = {} + else: + self._headers = self._session.get_headers() + self._cookies = self._session.get_cookies() + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def _update_from_data(self, data: ProjectDict): + self.id = int(data.get("id", self.id)) + self.url = f"https://scratch.mit.edu/projects/{self.id}" + if author := data.get("author"): + self.author_name = author.get("username", self.author_name) + self.author_name = data.get("username", self.author_name) + self.comments_allowed = data.get("comments_allowed", self.comments_allowed) + self.instructions = data.get("instructions", self.instructions) + self.notes = data.get("description", self.notes) + if history := data.get("history"): + self.created = history.get("created", self.created) + self.last_modified = history.get("modified", self.last_modified) + self.share_date = history.get("shared", self.share_date) + self.thumbnail_url = data.get("image", self.thumbnail_url) + remix_data = data.get("remix", {}) + self.remix_parent = remix_data.get("parent") + self.remix_root = remix_data.get("root") + if stats := data.get("stats"): + self.favorites = stats.get("favorites", self.favorites) + self.loves = stats.get("loves", self.loves) + self.remix_count = stats.get("remixes", self.remix_count) + self.views = stats.get("views", self.views) + self.title = data.get("title", self.title) + self.project_token = data.get("project_token", None) + return "code" not in data + + def __rich__(self): + from rich.panel import Panel + from rich.table import Table + from rich import box + from rich.markup import escape + + url = f"[link={self.url}]{self.title}[/]" + ret = Table.grid(expand=True) + ret.add_column(ratio=1) + ret.add_column(ratio=3) + info = Table(box=box.SIMPLE) + info.add_column(url, overflow="fold") + info.add_column(f"#{self.id}", overflow="fold") + info.add_row("By", self.author_name) + info.add_row("Created", escape(self.created)) + info.add_row("Shared", escape(self.share_date)) + info.add_row("Modified", escape(self.last_modified)) + info.add_row("Comments allowed", escape(str(self.comments_allowed))) + info.add_row("Loves", str(self.loves)) + info.add_row("Faves", str(self.favorites)) + info.add_row("Remixes", str(self.remix_count)) + info.add_row("Views", str(self.views)) + desc = Table(box=box.SIMPLE) + desc.add_row("Instructions", escape(self.instructions)) + desc.add_row("Notes & Credits", escape(self.notes)) + ret.add_row(Panel(info, title=url), Panel(desc, title="Description")) + return ret + + @property + def embed_url(self): + """ + Returns: + the url of the embed of the project + """ + return f"{self.url}/embed" + + def remixes(self, *, limit=40, offset=0) -> list[Project]: + """ + Returns: + list: A list containing the remixes of the project, each project is represented by a Project object. + """ + response = commons.api_iterative(f"https://api.scratch.mit.edu/projects/{self.id}/remixes", limit=limit, offset=offset) + return commons.parse_object_list(response, Project, self._session) + + def is_shared(self): + """ + Returns: + boolean: Returns whether the project is currently shared + """ + try: + p = get_project(self.id) + return isinstance(p, Project) + except exceptions.ProjectNotFound: + return False + + def raw_json_or_empty(self) -> dict[str, Any]: + return empty_project_json + + def create_remix(self, *, title=None, project_json=None) -> Project: + """ + Creates a project on the Scratch website. + + Warning: + Don't spam this method - it WILL get you banned from Scratch. + To prevent accidental spam, a rate limit (5 projects per minute) is implemented for this function. + """ + session = self._assert_auth() + if title is None: + if "title" in self.__dict__: + title = self.title + " remix" + else: + title = " remix" + if project_json is None: + project_json = self.raw_json_or_empty() + if len(CREATE_PROJECT_USES) < 5: + CREATE_PROJECT_USES.insert(0, time.time()) + else: + if CREATE_PROJECT_USES[-1] < time.time() - 300: + CREATE_PROJECT_USES.pop() + else: + raise exceptions.BadRequest( + "Rate limit for remixing Scratch projects exceeded.\nThis rate limit is enforced by scratchattach, not by the Scratch API.\nFor security reasons, it cannot be turned off.\n\nDon't spam-create projects, it WILL get you banned." + ) + CREATE_PROJECT_USES.insert(0, time.time()) + params = {"is_remix": "1", "original_id": self.id, "title": title} + response = requests.post( + "https://projects.scratch.mit.edu/", params=params, cookies=self._cookies, headers=self._headers, json=project_json + ).json() + _project = session.connect_project(response["content-name"]) + _project.parent_title = base64.b64decode(response["content-title"]).decode("utf-8").split(" remix")[0] + return _project + + def load_description(self): + """ + Gets the instructions of the unshared project. Requires authentication. + + Warning: + It's unclear if Scratch allows using this method. This method will create a remix of the unshared project using your account. + """ + self._assert_auth() + new_project = self.create_remix(project_json=empty_project_json) + self.instructions = new_project.instructions + self.title = new_project.parent_title or "" + + +@dataclass +class Project(PartialProject): + """ + Represents a Scratch project. + """ + + def __repr__(self): + return f"-P {self.id} ({self.title})" + + def __str__(self): + return repr(self) + + @property + def thumbnail(self) -> bytes: + with requests.no_error_handling(): + return requests.get(self.thumbnail_url).content + + def _assert_permission(self): + session = self._assert_auth() + if session.username != self.author_name: + raise exceptions.Unauthorized("You need to be authenticated as the profile owner to do this.") + + def load_description(self): + self.update() + + def download(self, *, filename=None, dir="."): + """ + Downloads the project json to the given directory. + + Args: + filename (str): The name that will be given to the downloaded file. + dir (str): The path of the directory the file will be saved in. + """ + try: + if filename is None: + filename = str(self.id) + if not (dir.endswith("/") or dir.endswith("\\")): + dir += "/" + self.update() + response = requests.get(f"https://projects.scratch.mit.edu/{self.id}?token={self.project_token}", timeout=10) + filename = filename.removesuffix(".sb3") + with open(f"{dir}{filename}.sb3", "wb") as f: + f.write(response.content) + except Exception as exc: + raise exceptions.FetchError("Method only works for projects created with Scratch 3") from exc + + @deprecated("Use raw_json instead") + def get_json(self) -> str: + """ + Downloads the project json and returns it as a string + """ + try: + self.update() + response = requests.get(f"https://projects.scratch.mit.edu/{self.id}?token={self.project_token}", timeout=10) + return response.text + except Exception as exc: + raise exceptions.FetchError("Method only works for projects created with Scratch 3") from exc + + def body(self) -> editor.Project: + """ + Method only works for project created with Scratch 3. + + Returns: + scratchattach.editor.Project: The contents of the project as editor Project object + """ + raw_json = self.raw_json() + return editor.Project.from_json(raw_json) + + def raw_json(self): + """ + Method only works for project created with Scratch 3. + + Returns: + dict: The raw project JSON as decoded Python dictionary + """ + try: + self.update() + except Exception as e: + raise exceptions.FetchError(f"You're not authorized for accessing {self}.\nException: {e}") + with requests.no_error_handling(): + resp = requests.get(f"https://projects.scratch.mit.edu/{self.id}?token={self.project_token}", timeout=10) + try: + return resp.json() + except json.JSONDecodeError: + with zipfile.ZipFile(BytesIO(resp.content)) as zipf: + return json.load(zipf.open("project.json")) + + def raw_json_or_empty(self): + return self.raw_json() + + def creator_agent(self): + """ + Method only works for project created with Scratch 3. + + Returns: + str: The user agent of the browser that this project was saved with. + """ + return self.raw_json()["meta"]["agent"] + + def set_body(self, project_body: editor.Project): + """ + Sets the project's contents You can use this to upload projects to the Scratch website. + Returns a dict with Scratch's raw JSON API response. + + Args: + project_body (scratchattach.ProjectBody): A ProjectBody object containing the contents of the project + """ + self._assert_permission() + return self.set_json(project_body.to_json()) + + def set_json(self, json_data): + """ + Sets the project json. You can use this to upload projects to the Scratch website. + Returns a dict with Scratch's raw JSON API response. + + Args: + json_data (dict or JSON): The new project JSON as encoded JSON object or as dict + """ + self._assert_permission() + if not isinstance(json_data, dict): + json_data = json.loads(json_data) + return requests.put( + f"https://projects.scratch.mit.edu/{self.id}", headers=self._headers, cookies=self._cookies, json=json_data + ).json() + + def upload_json_from(self, project_id: int | str): + """ + Uploads the project json from the project with the given id to the project represented by this Project object + """ + self._assert_auth() + other_project = self._session.connect_project(project_id) + self.set_json(other_project.raw_json()) + + def author(self) -> user.User: + """ + Returns: + scratchattach.user.User: An object representing the Scratch user who created this project. + """ + return self._make_linked_object("username", self.author_name, user.User, exceptions.UserNotFound) + + def studios(self, *, limit=40, offset=0): + """ + Returns: + list: A list containing the studios this project is in, each studio is represented by a Studio object. + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/studios", + limit=limit, + offset=offset, + add_params=f"&cachebust={random.randint(0, 9999)}", + ) + return commons.parse_object_list(response, studio.Studio, self._session) + + def comments(self, *, limit=40, offset=0) -> list["comment.Comment"]: + """ + Returns the comments posted on the project (except for replies. To get replies use :meth:`scratchattach.project.Project.comment_replies`). + + Keyword Arguments: + page: The page of the comments that should be returned. + limit: Max. amount of returned comments. + + Returns: + list: A list containing the requested comments as Comment objects. + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/comments/", + limit=limit, + offset=offset, + add_params=f"&cachebust={random.randint(0, 9999)}", + _headers=self._headers, + cookies=self._cookies, + ) + for i in response: + i["source"] = "project" + i["source_id"] = self.id + return commons.parse_object_list(response, comment.Comment, self._session) + + def comment_replies(self, *, comment_id, limit=40, offset=0): + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/comments/{comment_id}/replies/", + limit=limit, + offset=offset, + add_params=f"&cachebust={random.randint(0, 9999)}", + _headers=self._headers, + cookies=self._cookies, + ) + for x in response: + x["parent_id"] = comment_id + x["source"] = "project" + x["source_id"] = self.id + return commons.parse_object_list(response, comment.Comment, self._session) + + def comment_by_id(self, comment_id): + """ + Returns: + scratchattach.comments.Comment: A Comment object representing the requested comment. + """ + data = requests.get( + f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/comments/{comment_id}", + headers=self._headers, + cookies=self._cookies, + ).json() + if data is None or data.get("code") == "NotFound": + raise exceptions.CommentNotFound(f"Cannot find comment #{comment_id} on -P {self.id} by -U {self.author_name}") + _comment = comment.Comment( + id=data["id"], _session=self._session, source=comment.CommentSource.PROJECT, source_id=self.id + ) + _comment._update_from_data(data) + return _comment + + def love(self): + """ + Posts a love on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + session = self._assert_auth() + r = requests.post( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/loves/user/{session.username}", + headers=self._headers, + cookies=self._cookies, + ).json() + if "userLove" in r: + if r["userLove"] is False: + self.love() + else: + raise exceptions.APIError(str(r)) + + def unlove(self): + """ + Removes the love from this project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + session = self._assert_auth() + r = requests.delete( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/loves/user/{session.username}", + headers=self._headers, + cookies=self._cookies, + ).json() + if "userLove" in r: + if r["userLove"] is True: + self.unlove() + else: + raise exceptions.APIError(str(r)) + + def favorite(self): + """ + Posts a favorite on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + session = self._assert_auth() + r = requests.post( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/favorites/user/{session.username}", + headers=self._headers, + cookies=self._cookies, + ).json() + if "userFavorite" in r: + if r["userFavorite"] is False: + self.favorite() + else: + raise exceptions.APIError(str(r)) + + def unfavorite(self): + """ + Removes the favorite from this project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + session = self._assert_auth() + r = requests.delete( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/favorites/user/{session.username}", + headers=self._headers, + cookies=self._cookies, + ).json() + if "userFavorite" in r: + if r["userFavorite"] is True: + self.unfavorite() + else: + raise exceptions.APIError(str(r)) + + def post_view(self): + """ + Increases the project's view counter by 1. Doesn't require a login. + """ + requests.post(f"https://api.scratch.mit.edu/users/{self.author_name}/projects/{self.id}/views/", headers=headers) + + def set_fields(self, fields_dict, *, use_site_api=False): + """ + Sets fields. By default, ueses the api.scratch.mit.edu/projects/xxx/ PUT API. + + Keyword Arguments: + use_site_api (bool): + When enabled, the fields are set using the scratch.mit.edu/site-api API. + This function allows setting more fields than Project.set_fields. + For example, you can also share / unshare the project by setting the "shared" field. + According to the Scratch team, this API is deprecated. As of 2024 it's still fully functional though. + """ + self._assert_permission() + if use_site_api: + r = requests.put( + f"https://scratch.mit.edu/site-api/projects/all/{self.id}", + headers=self._headers, + cookies=self._cookies, + json=fields_dict, + ).json() + else: + r = requests.put( + f"https://api.scratch.mit.edu/projects/{self.id}", + headers=self._headers, + cookies=self._cookies, + json=fields_dict, + ).json() + return self._update_from_data(r) + + def turn_off_commenting(self): + """ + Disables commenting on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + data = {"comments_allowed": False} + self.set_fields(data) + + def turn_on_commenting(self): + """ + Enables commenting on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + data = {"comments_allowed": True} + self.set_fields(data) + + def toggle_commenting(self): + """ + Switches commenting on / off on the project (If comments are on, they will be turned off, else they will be turned on). You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + data = {"comments_allowed": not self.comments_allowed} + self.set_fields(data) + + def share(self): + """ + Shares the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self._assert_permission() + requests.put( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/share/", headers=self._json_headers, cookies=self._cookies + ) + + def unshare(self): + """ + Unshares the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self._assert_permission() + requests.put( + f"https://api.scratch.mit.edu/proxy/projects/{self.id}/unshare/", headers=self._json_headers, cookies=self._cookies + ) + + ' doesn\'t work. the API\'s response is valid (no errors), but the fields don\'t change\n def move_to_trash(self):\n """\n Moves the project to trash folder. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project`\n """\n self.set_fields({"id":int(self.id), "visibility": "trshbyusr", "isPublished" : False}, use_site_api=True)' + + def set_thumbnail(self, *, file): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self._assert_permission() + with open(file, "rb") as f: + thumbnail = f.read() + requests.post( + f"https://scratch.mit.edu/internalapi/project/thumbnail/{self.id}/set/", + data=thumbnail, + headers=self._headers, + cookies=self._cookies, + ) + + def delete_comment(self, *, comment_id): + """ + Deletes a comment by its ID. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + comment_id: The id of the comment that should be deleted + """ + self._assert_permission() + return requests.delete( + f"https://api.scratch.mit.edu/proxy/comments/project/{self.id}/comment/{comment_id}/", + headers=self._headers, + cookies=self._cookies, + ) + + def report_comment(self, *, comment_id): + """ + Reports a comment by its ID to the Scratch team. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + comment_id: The id of the comment that should be reported + """ + self._assert_auth() + return requests.delete( + f"https://api.scratch.mit.edu/proxy/comments/project/{self.id}/comment/{comment_id}/report", + headers=self._headers, + cookies=self._cookies, + ) + + def post_comment(self, content, *, parent_id="", commentee_id=""): + """ + Posts a comment on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + content: Content of the comment that should be posted + + Keyword Arguments: + parent_id: ID of the comment you want to reply to. If you don't want to mention a user, don't put the argument. + commentee_id: ID of the user that will be mentioned in your comment and will receive a message about your comment. If you don't want to mention a user, don't put the argument. + + Returns: + scratchattach.comments.Comment: Comment object representing the posted comment. + """ + self._assert_auth() + data = {"commentee_id": commentee_id, "content": str(content), "parent_id": parent_id} + r = json.loads( + requests.post( + f"https://api.scratch.mit.edu/proxy/comments/project/{self.id}/", + headers=self._json_headers | {"referer": "https://scratch.mit.edu/projects/" + str(self.id) + "/"}, + cookies=self._cookies, + data=json.dumps(data), + ).text + ) + if "id" not in r: + raise exceptions.CommentPostFailure(r) + _comment = comment.Comment(id=r["id"], _session=self._session, source=comment.CommentSource.PROJECT, source_id=self.id) + _comment._update_from_data(r) + return _comment + + def reply_comment(self, content, *, parent_id, commentee_id=""): + """ + Posts a reply to a comment on the project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + content: Content of the comment that should be posted + + Warning: + Only replies to top-level comments are shown on the Scratch website. Replies to replies are actually replies to the corresponding top-level comment in the API. + + Therefore, parent_id should be the comment id of a top level comment. + + Keyword Arguments: + parent_id: ID of the comment you want to reply to + commentee_id: ID of the user you are replying to + """ + return self.post_comment(content, parent_id=parent_id, commentee_id=commentee_id) + + def set_title(self, text): + """ + Changes the projects title. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self.set_fields({"title": text}) + + def set_instructions(self, text): + """ + Changes the projects instructions. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self.set_fields({"instructions": text}) + + def set_notes(self, text): + """ + Changes the projects notes and credits. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + """ + self.set_fields({"description": text}) + + @deprecated("Deprecated because ScratchDB is down indefinitely.") + def ranks(self): + """ + Gets information about the project's ranks. Fetched from ScratchDB. + + Warning: + This method is deprecated because ScratchDB is down indefinitely. + + Returns: + dict: A dict containing the project's ranks. If the ranks aren't available, all values will be -1. + """ + return requests.get(f"https://scratchdb.lefty.one/v3/project/info/{self.id}").json()["statistics"]["ranks"] + + def moderation_status(self, *, reload: bool = False): + """ + Gets information about the project's moderation status. Fetched from jeffalo's API. + + Returns: + str: The moderation status of the project. + + These moderation statuses exist: + + safe: The project was reviewed by the Scratch team and was considered safe for everyone. + + notsafe: The project was reviewed by the Scratch team and was considered not safe for everyone (nfe). It can't appear in search results, on the explore page and on the front page. + + notreviewed: The project hasn't been reviewed yet. + + no_remixes: Unable to fetch the project's moderation status. + """ + if self._moderation_status and (not reload): + return self._moderation_status + try: + return requests.get(f"https://jeffalo.net/api/nfe/?project={self.id}").json()["status"] + except Exception as exc: + raise exceptions.FetchError from exc + + def visibility(self): + """ + Returns info about the project's visibility. Requires authentication. + """ + session = self._assert_auth() + return requests.get( + f"https://api.scratch.mit.edu/users/{session.username}/projects/{self.id}/visibility", + headers=self._headers, + cookies=self._cookies, + ).json() + + +def get_project(project_id) -> Project: + """ + Gets a project without logging in. + + Args: + project_id (int): Project id of the requested project + + Returns: + scratchattach.project.Project: An object representing the requested project. + + Warning: + Any methods that require authentication (like project.love) will not work on the returned object. + + If you want to use these methods, get the project with :meth:`scratchattach.session.Session.connect_project` instead. + """ + warnings.warn( + "For methods that require authentication, use session.connect_project instead of get_project.\nIf you want to remove this warning, use `warnings.filterwarnings('ignore', category=scratchattach.ProjectAuthenticationWarning)`.\nTo ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use `warnings.filterwarnings('ignore', category=scratchattach.GetAuthenticationWarning)`.", + exceptions.ProjectAuthenticationWarning, + ) + return commons._get_object("id", project_id, Project, exceptions.ProjectNotFound) + + +def search_projects(*, query="", mode="trending", language="en", limit=40, offset=0): + """ + Uses the Scratch search to search projects. + + Keyword arguments: + query (str): The query that will be searched. + mode (str): Has to be one of these values: "trending", "popular" or "recent". Defaults to "trending". + language (str): A language abbreviation, defaults to "en". (Depending on the language used on the Scratch website, Scratch displays you different results.) + limit (int): Max. amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: List that contains the search results. + """ + if not query: + raise ValueError("The query can't be empty for search") + response = commons.api_iterative( + "https://api.scratch.mit.edu/search/projects", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}&q={query}", + ) + return commons.parse_object_list(response, Project) + + +def explore_projects(*, query="*", mode="trending", language="en", limit=40, offset=0): + """ + Gets projects from the explore page. + + Keyword arguments: + query (str): Specifies the tag of the explore page. To get the projects from the "All" tag, set this argument to "*". + mode (str): Has to be one of these values: "trending", "popular" or "recent". Defaults to "trending". + language (str): A language abbreviation, defaults to "en". (Depending on the language used on the Scratch website, Scratch displays you different explore pages.) + limit (int): Max. amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: List that contains the explore page projects. + """ + if not query: + raise ValueError("The query can't be empty for search") + response = commons.api_iterative( + "https://api.scratch.mit.edu/explore/projects", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}&q={query}", + ) + return commons.parse_object_list(response, Project) diff --git a/scratchattach/sync_api/site/session.py b/scratchattach/sync_api/site/session.py new file mode 100644 index 00000000..69e9e9c3 --- /dev/null +++ b/scratchattach/sync_api/site/session.py @@ -0,0 +1,1389 @@ +"""Session class and login function""" + +from __future__ import annotations +from types import TracebackType +import base64 +import bs4 +import datetime +import hashlib +import json +import pathlib +import random +import re +import time +import warnings +import zlib +from dataclasses import dataclass, field +from typing import Literal, Optional, TypeVar, TYPE_CHECKING, overload, Any, Union, cast, Self +from contextlib import contextmanager +from threading import local +from scratchattach import editor + +Type = type +if TYPE_CHECKING: + from _typeshed import FileDescriptorOrPath, SupportsRead + from scratchattach.cloud._base import BaseCloud + + T = TypeVar("T", bound=BaseCloud) +else: + T = TypeVar("T") +from bs4 import BeautifulSoup, Tag +from typing_extensions import deprecated +from . import activity, classroom, forum, studio, user, project, backpack_asset, alert +from . import typed_dicts +from ._base import BaseSiteComponent, api_iterative +from scratchattach.cloud import cloud, _base +from scratchattach.eventhandlers import message_events, filterbot +from scratchattach.other import other_apis +from scratchattach.utils import commons, exceptions +from scratchattach.utils.commons import headers, empty_project_json, webscrape_count, get_class_sort_mode +from scratchattach._shared import http as shared_http +from ..primitives import http +from .browser_cookies import Browser, ANY, cookies_from_browser + +ratelimit_cache: dict[str, list[float]] = {} + + +def enforce_ratelimit(__type: str, name: str, amount: int = 5, duration: int = 60) -> None: + cache = ratelimit_cache + cache.setdefault(__type, []) + uses = cache[__type] + while uses and uses[-1] < time.time() - duration: + uses.pop() + if len(uses) < amount: + uses.insert(0, time.time()) + return + raise exceptions.RateLimitedError( + f"Rate limit for {name} exceeded.\nThis rate limit is enforced by scratchattach, not by the Scratch API.\nFor security reasons, it cannot be turned off.\n\nDon't spam-create studios or similar, it WILL get you banned." + ) + + +C = TypeVar("C", bound=BaseSiteComponent) + + +class UnauthSession: + http_session: http._HTTPSession + + +@dataclass +class Session(BaseSiteComponent[typed_dicts.SessionDict]): + """ + Represents a Scratch log in / session. Stores authentication data (session id and xtoken). + + Attributes: + id: The session id associated with the login + username: The username associated with the login + xtoken: The xtoken associated with the login + email: The email address associated with the logged in account + new_scratcher: True if the associated account is a new Scratcher + mute_status: Information about commenting restrictions of the associated account + banned: Returns True if the associated account is banned + """ + + http_session: http._HTTPSession = field(repr=False, kw_only=True) + username: str = field(repr=False, default="") + _user: Optional[user.User] = field(repr=False, default=None) + id: str = field(repr=False, default="") + session_string: Optional[str] = field(repr=False, default=None) + xtoken: Optional[str] = field(repr=False, default=None) + email: Optional[str] = field(repr=False, default=None) + new_scratcher: bool = field(repr=False, default=False) + mute_status: Any = field(repr=False, default=None) + banned: bool = field(repr=False, default=False) + time_created: datetime.datetime = field(repr=False, default=datetime.datetime.fromtimestamp(0.0)) + language: str = field(repr=False, default="en") + has_outstanding_email_confirmation: bool = field(repr=False, default=False) + is_teacher: bool = field(repr=False, default=False) + is_teacher_invitee: bool = field(repr=False, default=False) + ocular_token: Optional[str] = field(repr=False, default=None) + _session: Session | UnauthSession = field(kw_only=True, init=False) + + def __str__(self) -> str: + return f"-L {self.username}" + + def __rich__(self): + from rich.panel import Panel + from rich.table import Table + from rich import box + from rich.markup import escape + + ret = Table( + f"[link={self.connect_linked_user().url}]{escape(self.username)}[/]", f"Created: {self.time_created}", expand=True + ) + ret.add_row("Email", escape(str(self.email))) + ret.add_row("Language", escape(str(self.language))) + ret.add_row("Mute status", escape(str(self.mute_status))) + ret.add_row("New scratcher?", str(self.new_scratcher)) + ret.add_row("Banned?", str(self.banned)) + ret.add_row("Has outstanding email confirmation?", str(self.has_outstanding_email_confirmation)) + ret.add_row("Is teacher invitee?", str(self.is_teacher_invitee)) + ret.add_row("Is teacher?", str(self.is_teacher)) + return ret + + @property + def _username(self) -> str: + return self.username + + def __post_init__(self): + self.update_function = shared_http.HTTPMethod.POST + self.update_api = "https://scratch.mit.edu/session" + self._headers = dict(headers) + try: + self.id = json.loads(self.id) + except json.JSONDecodeError: + pass + self._cookies = { + "scratchsessionsid": self.id, + "scratchcsrftoken": "a", + "scratchlanguage": "en", + "accept": "application/json", + "Content-Type": "application/json", + } + self._update_http_cookies_and_headers() + if self.id: + self._process_session_id() + self._session = self + + def _enter(self) -> Self: + self.http_session.__enter__() + return self + + def _exit(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None) -> None: + self.http_session.__exit__(exc_type, exc_val, exc_tb) + + def _update_from_data(self, data: typed_dicts.SessionDict): + self.xtoken = data["user"]["token"] + self._headers["X-Token"] = self.xtoken + self.has_outstanding_email_confirmation = data["flags"]["has_outstanding_email_confirmation"] + self.email = data["user"]["email"] + self.new_scratcher = data["permissions"]["new_scratcher"] + self.is_teacher = data["permissions"]["educator"] + self.is_teacher_invitee = data["permissions"]["educator_invitee"] + self.mute_status = data["permissions"]["mute_status"] + self.username = data["user"]["username"] + self.banned = data["user"]["banned"] + if self.banned: + warnings.warn( + f"Warning: The account {self.username} you logged in to is BANNED. Some features may not work properly." + ) + if self.has_outstanding_email_confirmation: + warnings.warn( + f"Warning: The account {self.username} you logged is not email confirmed. Some features may not work properly." + ) + return True + + def _process_session_id(self): + assert self.id + data, self.time_created = decode_session_id(self.id) + self.username = data["username"] + self.xtoken = data["token"] + self._headers["X-Token"] = self.xtoken + self.language = data.get("_language", "en") + + def _assert_ocular_auth(self) -> str: + if not self.ocular_token: + raise ValueError( + f"No ocular token supplied for {self}! You can add one by using Session.set_ocular_token(YOUR_TOKEN)." + ) + return self.ocular_token + + def _update_http_cookies_and_headers(self): + self.http_session.clear_cookies() + self.http_session.update_cookies(self._cookies) + self.http_session.clear_headers() + self.http_session.update_headers(self._headers) + + def set_ocular_token(self, token: str): + self.ocular_token = token + + def connect_linked_user(self) -> user.User: + """ + Gets the user associated with the login / session. + + Warning: + The returned User object is cached. To ensure its attribute are up to date, you need to run .update() on it. + + Returns: + scratchattach.user.User: Object representing the user associated with the session. + """ + cached = hasattr(self, "_user") + if cached: + cached = self._user is not None + if not cached: + self._user = self.connect_user(self._username) + assert self._user is not None + return self._user + + def get_linked_user(self) -> user.User: + return self.connect_linked_user() + + def set_country(self, country: str = "Antarctica"): + """ + Sets the profile country of the session's associated user + + Arguments: + country (str): The country to relocate to + """ + with self.http_session.post( + "https://scratch.mit.edu/accounts/settings/", shared_http.options().data({"country": country}).value + ): + pass + + def resend_email(self, password: str): + """ + Sends a request to resend a confirmation email for this session's account + + Keyword arguments: + password (str): Password associated with the session (not stored) + """ + with self.http_session.post( + "https://scratch.mit.edu/accounts/email_change/", + shared_http.options().data({"email_address": self.get_new_email_address(), "password": password}).value, + ): + pass + + def get_new_email_address(self) -> str: + """ + Gets the (unconfirmed) email address that this session has requested to transfer to, if any, + otherwise the current address. + + Returns: + str: The email that this session wants to switch to + """ + with self.http_session.get("https://scratch.mit.edu/accounts/email_change/") as response: + soup = BeautifulSoup(response.text(), "html.parser") + email = None + for label_span in soup.find_all("span", {"class": "label"}): + if not isinstance(label_span, Tag): + continue + if not isinstance(label_span.parent, Tag): + continue + if label_span.contents[0] == "New Email Address": + return label_span.parent.contents[-1].text.strip("\n ") + elif label_span.contents[0] == "Current Email Address": + email = label_span.parent.contents[-1].text.strip("\n ") + if email is None: + for label_span in soup.select("form#email-change span.current-email"): + email = label_span.text + assert email is not None + return email + + def logout(self): + """ + Sends a logout request to scratch. (Might not do anything, might log out this account on other ips/sessions.) + """ + with self.http_session.post("https://scratch.mit.edu/accounts/logout/"): + pass + + def set_featured_data(self, project_id: Optional[int] | Literal[""], project_label: Optional[int] | Literal[""] = None): + """ + Sends a request to change your featured project area. + + Positional arguments: + project_id: None -> don't change; empty string -> set to latest project (this is what most accounts have); int -> set the featured project to the one with the corresponding ID. If you do not own that project, an error is raised. + project_lavel: None -> don't change; empty string -> "Featured project"; 0 -> "Featured Tutorial"; 1 -> "Work in progress"; 2 -> "Remix this!"; 3 -> "My favorite things"; 4 -> "Why I scratch" + + Returns: + list: List that contains all messages as Activity objects. + + """ + payload: dict[str, int | str] = {} + if project_label is not None: + payload["featured_project_label"] = str(project_label) + if project_id is not None: + payload["featured_project"] = project_id + with self.http_session.put( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", shared_http.options().json(payload).value + ) as response: + data = response.json() + if errors := data.get("errors"): + raise Exception(f"Backend responded with error: {(errors[0] if len(errors) == 1 else errors)}") + return data + + @property + def ocular_headers(self) -> dict[str, str]: + return { + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36", + "referer": "https://ocular.jeffalo.net/", + "authorization": self._assert_ocular_auth(), + } + + def get_ocular_status(self) -> typed_dicts.OcularUserDict: + self._assert_ocular_auth() + with self.http_session.get( + "https://my-ocular.jeffalo.net/auth/me", + shared_http.options().disregard_default_headers().disregard_default_cookies().headers(self.ocular_headers).value, + ) as response: + return cast(typed_dicts.OcularUserDict, response.json()) + + def set_ocular_status(self, status: Optional[str] = None, color: Optional[str] = None) -> None: + self._assert_ocular_auth() + old = self.get_ocular_status() + payload = {"color": color or old["color"], "status": status or old["status"]} + with self.http_session.put( + f"https://my-ocular.jeffalo.net/api/user/{old['name']}", + shared_http.options() + .disregard_default_headers() + .disregard_default_cookies() + .headers(self.ocular_headers) + .json(payload) + .value, + ) as response: + assert response.json() == {"ok": "user updated"}, ( + f"Error occured on setting ocular status. auth/me response: {old}" + ) + + def messages(self, *, limit: int = 40, offset: int = 0, date_limit=None, filter_by=None) -> list[activity.Activity]: + """ + Returns the messages. + + Keyword arguments: + limit, offset, date_limit + filter_by (str or None): Can either be None (no filter), "comments", "projects", "studios" or "forums" + + Returns: + list: List that contains all messages as Activity objects. + """ + add_params = "" + if date_limit is not None: + add_params += f"&dateLimit={date_limit}" + if filter_by is not None: + add_params += f"&filter={filter_by}" + data: list[Any] = api_iterative( + self, + f"https://api.scratch.mit.edu/users/{self._username}/messages", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + add_params=add_params, + ) + return activity.Activity.parse_object_list(data, self) + + def admin_messages(self, *, limit=40, offset=0) -> list[dict]: + """ + Returns your messages sent by the Scratch team (alerts). + """ + return commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/messages/admin", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + ) + + def classroom_alerts( + self, _classroom: Optional[classroom.Classroom | int] = None, mode: str = "Last created", page: Optional[int] = None + ): + """ + Load and parse admin alerts, optionally for a specific class, using https://scratch.mit.edu/site-api/classrooms/alerts/ + + Returns: + list[alert.EducatorAlert]: A list of parsed EducatorAlert objects + """ + if isinstance(_classroom, classroom.Classroom): + _classroom = _classroom.id + if _classroom is None: + _classroom_str = "" + else: + _classroom_str = f"{_classroom}/" + ascsort, descsort = get_class_sort_mode(mode) + with self.http_session.get( + f"https://scratch.mit.edu/site-api/classrooms/alerts/{_classroom_str}", + shared_http.options().params({"page": page, "ascsort": ascsort, "descsort": descsort}).value, + ) as response: + data = response.json() + alerts = [alert.EducatorAlert.from_json(alert_data, self) for alert_data in data] + return alerts + + def clear_messages(self): + """ + Clears all messages. + """ + return requests.post( + "https://scratch.mit.edu/site-api/messages/messages-clear/", + headers=self._headers, + cookies=self._cookies, + timeout=10, + ).text + + def message_count(self) -> int: + """ + Returns the message count. + + Returns: + int: message count + """ + return json.loads( + requests.get( + f"https://scratch.mit.edu/messages/ajax/get-message-count/", + headers=self._headers, + cookies=self._cookies, + timeout=10, + ).text + )["msg_count"] + + def feed(self, *, limit=20, offset=0, date_limit=None) -> list[activity.Activity]: + """ + Returns the "What's happening" section (frontpage). + + Returns: + list: List that contains all "What's happening" entries as Activity objects + """ + add_params = "" + if date_limit is not None: + add_params = f"&dateLimit={date_limit}" + data = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/following/users/activity", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + add_params=add_params, + ) + return commons.parse_object_list(data, activity.Activity, self) + + def get_feed(self, *, limit=20, offset=0, date_limit=None): + return self.feed(limit=limit, offset=offset, date_limit=date_limit) + + def loved_by_followed_users(self, *, limit=40, offset=0) -> list[project.Project]: + """ + Returns the "Projects loved by Scratchers I'm following" section (frontpage). + + Returns: + list: List that contains all "Projects loved by Scratchers I'm following" + entries as Project objects + """ + data = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/following/users/loves", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + ) + return commons.parse_object_list(data, project.Project, self) + + def shared_by_followed_users(self, *, limit=40, offset=0) -> list[project.Project]: + """ + Returns the "Projects by Scratchers I'm following" section (frontpage). + This section is only visible to old accounts (until ~2018). + For newer users, this method will always return an empty list. + + Returns: + list: List that contains all "Projects by Scratchers I'm following" + entries as Project objects + """ + data = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/following/users/projects", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + ) + ret = commons.parse_object_list(data, project.Project, self) + if not ret: + warnings.warn( + f"`shared_by_followed_users` got empty list `[]`. Note that this method is not supported for accounts made after 2018." + ) + return ret + + def in_followed_studios(self, *, limit=40, offset=0) -> list["project.Project"]: + """ + Returns the "Projects in studios I'm following" section (frontpage). + This section is only visible to old accounts (until ~2018) + For newer users, this method will always return an empty list. + + Returns: + list: List that contains all "Projects in studios I'm following" section" + entries as Project objects + """ + data = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self._username}/following/studios/projects", + limit=limit, + offset=offset, + _headers=self._headers, + cookies=self._cookies, + ) + ret = commons.parse_object_list(data, project.Project, self) + if not ret: + warnings.warn( + f"`in_followed_studios` got empty list `[]`. Note that this method is not supported for accounts made after 2018." + ) + return ret + + def connect_empty_project_pb(self) -> editor.Project: + pb = editor.Project.from_json(empty_project_json) + pb._session = self + return pb + + def connect_pb_from_dict(self, project_json: dict) -> editor.Project: + pb = editor.Project.from_json(project_json) + pb._session = self + return pb + + def connect_pb_from_file(self, path_to_file) -> editor.Project: + pb = editor.Project.from_sb3(path_to_file) + pb._session = self + return pb + + @staticmethod + def download_asset(asset_id_with_file_ext, *, filename: Optional[str] = None, fp=""): + if not (fp.endswith("/") or fp.endswith("\\")): + fp = fp + "/" + try: + if filename is None: + filename = str(asset_id_with_file_ext) + response = requests.get("https://assets.scratch.mit.edu/" + str(asset_id_with_file_ext), timeout=10) + open(f"{fp}{filename}", "wb").write(response.content) + except Exception: + raise exceptions.FetchError("Failed to download asset") + + def upload_asset(self, asset_content, *, asset_id=None, file_ext=None): + data = asset_content if isinstance(asset_content, bytes) else open(asset_content, "rb").read() + if isinstance(asset_content, str): + file_ext = pathlib.Path(asset_content).suffix + file_ext = file_ext.replace(".", "") + if asset_id is None: + asset_id = hashlib.md5(data).hexdigest() + requests.post( + f"https://assets.scratch.mit.edu/{asset_id}.{file_ext}", + headers=self._headers, + cookies=self._cookies, + data=data, + timeout=10, + ) + + def search_projects( + self, *, query: str = "", mode: str = "trending", language: str = "en", limit: int = 40, offset: int = 0 + ) -> list[project.Project]: + """ + Uses the Scratch search to search projects. + + Keyword arguments: + query (str): The query that will be searched. + mode (str): Has to be one of these values: "trending", "popular" or "recent". Defaults to "trending". + language (str): A language abbreviation, defaults to "en". + (Depending on the language used on the Scratch website, Scratch displays you different results.) + limit (int): Max. amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: List that contains the search results. + """ + query = f"&q={query}" if query else "" + response = commons.api_iterative( + f"https://api.scratch.mit.edu/search/projects", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}{query}", + ) + return commons.parse_object_list(response, project.Project, self) + + def explore_projects( + self, *, query: str = "*", mode: str = "trending", language: str = "en", limit: int = 40, offset: int = 0 + ) -> list[project.Project]: + """ + Gets projects from the explore page. + + Keyword arguments: + query (str): Specifies the tag of the explore page. + To get the projects from the "All" tag, set this argument to "*". + mode (str): Has to be one of these values: "trending", "popular" or "recent". + Defaults to "trending". + language (str): A language abbreviation, defaults to "en". + (Depending on the language used on the Scratch website, Scratch displays you different explore pages.) + limit (int): Max. amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: List that contains the explore page projects. + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/explore/projects", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}&q={query}", + ) + return commons.parse_object_list(response, project.Project, self) + + def search_studios( + self, *, query: str = "", mode: str = "trending", language: str = "en", limit: int = 40, offset: int = 0 + ) -> list[studio.Studio]: + query = f"&q={query}" if query else "" + response = commons.api_iterative( + f"https://api.scratch.mit.edu/search/studios", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}{query}", + ) + return commons.parse_object_list(response, studio.Studio, self) + + def explore_studios( + self, *, query: str = "", mode: str = "trending", language: str = "en", limit: int = 40, offset: int = 0 + ) -> list[studio.Studio]: + query = f"&q={query}" if query else "" + response = commons.api_iterative( + f"https://api.scratch.mit.edu/explore/studios", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}{query}", + ) + return commons.parse_object_list(response, studio.Studio, self) + + def create_project( + self, *, title: Optional[str] = None, project_json: dict = empty_project_json, parent_id=None + ) -> project.Project: + """ + Creates a project on the Scratch website. + + Warning: + Don't spam this method - it WILL get you banned from Scratch. + To prevent accidental spam, a rate limit (5 projects per minute) is implemented for this function. + """ + enforce_ratelimit("create_scratch_project", "creating Scratch projects") + if title is None: + title = f"Untitled-{random.randint(0, 1 << 16)}" + params = {"is_remix": "0" if parent_id is None else "1", "original_id": parent_id, "title": title} + response = requests.post( + "https://projects.scratch.mit.edu/", params=params, cookies=self._cookies, headers=self._headers, json=project_json + ).json() + return self.connect_project(response["content-name"]) + + def create_studio(self, *, title: Optional[str] = None, description: Optional[str] = None) -> studio.Studio: + """ + Create a studio on the scratch website + + Warning: + Don't spam this method - it WILL get you banned from Scratch. + To prevent accidental spam, a rate limit (5 studios per minute) is implemented for this function. + """ + enforce_ratelimit("create_scratch_studio", "creating Scratch studios") + if self.new_scratcher: + raise exceptions.Unauthorized(f"\nNew scratchers (like {self.username}) cannot create studios.") + response = requests.post("https://scratch.mit.edu/studios/create/", cookies=self._cookies, headers=self._headers) + studio_id = webscrape_count(response.json()["redirect"], "/studios/", "/") + new_studio = self.connect_studio(studio_id) + if title is not None: + new_studio.set_title(title) + if description is not None: + new_studio.set_description(description) + return new_studio + + def create_class(self, title: str, desc: str = "") -> classroom.Classroom: + """ + Create a class on the scratch website + + Warning: + Don't spam this method - it WILL get you banned from Scratch. + To prevent accidental spam, a rate limit (5 classes per minute) is implemented for this function. + """ + enforce_ratelimit("create_scratch_class", "creating Scratch classes") + if not self.is_teacher: + raise exceptions.Unauthorized(f"{self.username} is not a teacher; can't create class") + data = requests.post( + "https://scratch.mit.edu/classes/create_classroom/", + json={"title": title, "description": desc}, + headers=self._headers, + cookies=self._cookies, + ).json() + class_id = data[0]["id"] + return self.connect_classroom(class_id) + + def mystuff_counts(self) -> tuple[int, int, int]: + """ + Gets the number of shared projects, unshared projects, and studios as listed on the mystuff page, + and returns them in that order. + + Example usage: + shared, unshared, studios = sess.mystuff_counts() + print(f"You have {shared} shared projects, {unshared} unshared projects, and are in {studios} studios") + """ + with requests.no_error_handling(): + resp = requests.get("https://scratch.mit.edu/mystuff/", headers=self._headers, cookies=self._cookies) + soup = bs4.BeautifulSoup(resp.text, "html.parser") + shared_elem = soup.select_one("span[data-content='shared-count']") + unshared_elem = soup.select_one("span[data-content='unshared-count']") + gallery_elem = soup.select_one("span[data-content='gallery-count']") + assert shared_elem is not None + assert unshared_elem is not None + assert gallery_elem is not None + shared: str = shared_elem.text.strip() + unshared: str = unshared_elem.text.strip() + gallery: str = gallery_elem.text.strip() + return (int(shared), int(unshared), int(gallery)) + + def mystuff_projects( + self, filter_arg: str = "all", *, page: int = 1, sort_by: str = "", descending: bool = True + ) -> list[project.Project]: + """ + Gets the projects from the "My stuff" page. + + Args: + filter_arg (str): Possible values for this parameter are "all", "shared", "unshared" and "trashed" + + Keyword Arguments: + page (int): The page of the "My Stuff" projects that should be returned + sort_by (str): The key the projects should be sorted based on. Possible values for this parameter are "" (then the projects are sorted based on last modified), "view_count", love_count", "remixers_count" (then the projects are sorted based on remix count) and "title" (then the projects are sorted based on title) + descending (boolean): Determines if the element with the highest key value (the key is specified in the sort_by argument) should be returned first. Defaults to True. + + Returns: + list: A list with the projects from the "My Stuff" page, each project is represented by a Project object. + """ + if descending: + ascsort = "" + descsort = sort_by + else: + ascsort = sort_by + descsort = "" + try: + targets = requests.get( + f"https://scratch.mit.edu/site-api/projects/{filter_arg}/?page={page}&ascsort={ascsort}&descsort={descsort}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + projects = [] + for target in targets: + projects.append( + project.Project( + id=target["pk"], + _session=self, + author_name=self._username, + comments_allowed=None, + instructions=None, + notes=None, + created=target["fields"]["datetime_created"], + last_modified=target["fields"]["datetime_modified"], + share_date=target["fields"]["datetime_shared"], + thumbnail_url="https:" + target["fields"]["thumbnail_url"], + favorites=target["fields"]["favorite_count"], + loves=target["fields"]["love_count"], + remix_count=target["fields"]["remixers_count"], + views=target["fields"]["view_count"], + title=target["fields"]["title"], + comment_count=target["fields"]["commenters_count"], + ) + ) + return projects + except Exception: + raise exceptions.FetchError() + + def mystuff_studios( + self, filter_arg: str = "all", *, page: int = 1, sort_by: str = "", descending: bool = True + ) -> list[studio.Studio]: + if descending: + ascsort = "" + descsort = sort_by + else: + ascsort = sort_by + descsort = "" + try: + params: dict[str, Union[str, int]] = {"page": page, "ascsort": ascsort, "descsort": descsort} + targets = requests.get( + f"https://scratch.mit.edu/site-api/galleries/{filter_arg}/", + params=params, + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + studios = [] + for target in targets: + studios.append( + studio.Studio( + id=target["pk"], + _session=self, + title=target["fields"]["title"], + description=None, + host_id=target["fields"]["owner"]["pk"], + host_name=target["fields"]["owner"]["username"], + open_to_all=None, + comments_allowed=None, + image_url="https:" + target["fields"]["thumbnail_url"], + created=target["fields"]["datetime_created"], + modified=target["fields"]["datetime_modified"], + follower_count=None, + manager_count=None, + curator_count=target["fields"]["curators_count"], + project_count=target["fields"]["projecters_count"], + ) + ) + return studios + except Exception: + raise exceptions.FetchError() + + def mystuff_classes_counts(self) -> tuple[int, int]: + """ + Returns the number of open and ended classes owned by a teacher session. + If this is not a teacher session, NotATeacherError is raised + """ + with requests.no_error_handling(): + resp = requests.get("https://scratch.mit.edu/educators/classes/", headers=self._headers, cookies=self._cookies) + if resp.status_code == 403: + raise exceptions.NotATeacherError("Response 403 when getting educators/classes") + soup = BeautifulSoup(resp.text, "html.parser") + sidebar = soup.find("div", {"id": "sidebar", "class": "tabs-index"}) + if not sidebar: + return (0, 0) + count_elem = sidebar.find("span", {"data-content": "classroom-count"}) + ended_elem = sidebar.find("span", {"data-content": "closed-count"}) + if not count_elem or not ended_elem: + return (0, 0) + count = str(count_elem.text).strip() + ended_count = str(ended_elem.text).strip() + return (int(count), int(ended_count)) + + def mystuff_classes(self, mode: str = "Last created", page: Optional[int] = None) -> list[classroom.Classroom]: + if not self.is_teacher: + self.update() + if not self.is_teacher: + raise exceptions.Unauthorized(f"{self.username} is not a teacher; can't have classes") + ascsort, descsort = get_class_sort_mode(mode) + classes_data = requests.get( + "https://scratch.mit.edu/site-api/classrooms/all/", + params={"page": page, "ascsort": ascsort, "descsort": descsort}, + headers=self._headers, + cookies=self._cookies, + ).json() + classes = [] + for data in classes_data: + fields = data["fields"] + educator_pf = fields["educator_profile"] + classes.append( + classroom.Classroom( + id=data["pk"], + title=fields["title"], + classtoken=fields["token"], + datetime=datetime.datetime.fromisoformat(fields["datetime_created"]), + author=user.User(username=educator_pf["user"]["username"], id=educator_pf["user"]["pk"], _session=self), + _session=self, + ) + ) + return classes + + def mystuff_ended_classes(self, mode: str = "Last created", page: Optional[int] = None) -> list[classroom.Classroom]: + if not self.is_teacher: + raise exceptions.Unauthorized(f"{self.username} is not a teacher; can't have (deleted) classes") + ascsort, descsort = get_class_sort_mode(mode) + classes_data = requests.get( + "https://scratch.mit.edu/site-api/classrooms/closed/", + params={"page": page, "ascsort": ascsort, "descsort": descsort}, + headers=self._headers, + cookies=self._cookies, + ).json() + classes = [] + for data in classes_data: + fields = data["fields"] + educator_pf = fields["educator_profile"] + classes.append( + classroom.Classroom( + id=data["pk"], + title=fields["title"], + classtoken=fields["token"], + datetime=datetime.datetime.fromisoformat(fields["datetime_created"]), + author=user.User(username=educator_pf["user"]["username"], id=educator_pf["user"]["pk"], _session=self), + _session=self, + ) + ) + return classes + + def backpack(self, limit: int = 20, offset: int = 0) -> list[backpack_asset.BackpackAsset]: + """ + Lists the assets that are in the backpack of the user associated with the session. + + Returns: + list: List that contains the backpack items + """ + data = commons.api_iterative( + f"https://backpack.scratch.mit.edu/{self._username}", limit=limit, offset=offset, _headers=self._headers + ) + return commons.parse_object_list(data, backpack_asset.BackpackAsset, self) + + def delete_from_backpack(self, backpack_asset_id) -> backpack_asset.BackpackAsset: + """ + Deletes an asset from the backpack. + + Args: + backpack_asset_id: ID of the backpack asset that will be deleted + """ + return backpack_asset.BackpackAsset(id=backpack_asset_id, _session=self).delete() + + def become_scratcher_invite(self) -> dict: + """ + If you are a new Scratcher and have been invited for becoming a Scratcher, this API endpoint will provide + more info on the invite. + """ + return requests.get( + f"https://api.scratch.mit.edu/users/{self.username}/invites", headers=self._headers, cookies=self._cookies + ).json() + + @overload + def connect_cloud(self, project_id, *, cloud_class: type[T]) -> T: + """ + Connects to a cloud as logged-in user. + + Args: + project_id: + + Keyword arguments: CloudClass: The class that the returned object should be of. By default, this class is + scratchattach.cloud.ScratchCloud. + + Returns: Type[scratchattach.cloud._base.BaseCloud]: An object representing the cloud of a project. Can be of any + class inheriting from BaseCloud. + """ + + @overload + def connect_cloud(self, project_id) -> cloud.ScratchCloud: + """ + Connects to a cloud (by default Scratch's cloud) as logged-in user. + + Args: + project_id: + + Keyword arguments: CloudClass: The class that the returned object should be of. By default, this class is + scratchattach.cloud.ScratchCloud. + + Returns: Type[scratchattach.cloud._base.BaseCloud]: An object representing the cloud of a project. Can be of any + class inheriting from BaseCloud. + """ + + def connect_cloud(self, project_id, *, cloud_class: Optional[type[_base.BaseCloud]] = None) -> _base.BaseCloud: + cloud_class = cloud_class or cloud.ScratchCloud + return cloud_class(project_id=project_id, _session=self) + + def connect_scratch_cloud(self, project_id) -> cloud.ScratchCloud: + """ + Returns: + scratchattach.cloud.ScratchCloud: An object representing the Scratch cloud of a project. + """ + return cloud.ScratchCloud(project_id=project_id, _session=self) + + def connect_tw_cloud( + self, project_id, *, purpose="", contact="", cloud_host="wss://clouddata.turbowarp.org" + ) -> cloud.TwCloud: + """ + Returns: + scratchattach.cloud.TwCloud: An object representing the TurboWarp cloud of a project. + """ + return cloud.TwCloud(project_id=project_id, purpose=purpose, contact=contact, cloud_host=cloud_host, _session=self) + + def _make_linked_object( + self, identificator_name, identificator, __class: type[C], NotFoundException: type[Exception] + ) -> C: + """ + The Session class doesn't save the login in a ._session attribute, but IS the login ITSELF. + + Therefore, the _make_linked_object method has to be adjusted + to get it to work for in the Session class. + + Class must inherit from BaseSiteComponent + """ + return commons._get_object(identificator_name, identificator, __class, NotFoundException, self) + + def connect_user(self, username: str) -> user.User: + """ + Gets a user using this session, connects the session to the User object to allow authenticated actions + + Args: + username (str): Username of the requested user + + Returns: + scratchattach.user.User: An object that represents the requested user and allows you to perform actions on the user (like user.follow) + """ + return self._make_linked_object("username", username, user.User, exceptions.UserNotFound) + + @deprecated("Finding usernames by user ids has been fixed.") + def find_username_from_id(self, user_id: int) -> str: + """ + Warning: + Every time this functions is run, a comment on your profile is posted and deleted. Therefore you shouldn't run this too often. + + Returns: + str: The username that corresponds to the user id + """ + you = user.User(username=self.username, _session=self) + try: + comment = you.post_comment("scratchattach", commentee_id=int(user_id)) + except exceptions.CommentPostFailure: + raise exceptions.BadRequest( + "After posting a comment, you need to wait 10 seconds before you can connect users by id again." + ) + except exceptions.BadRequest: + raise exceptions.UserNotFound("Invalid user id") + except Exception as e: + raise e + you.delete_comment(comment_id=comment.id) + try: + username = comment.content.split('">@')[1] + username = username.split("")[0] + except IndexError: + raise exceptions.UserNotFound() + return username + + @deprecated("Finding usernames by user ids has been fixed.") + def connect_user_by_id(self, user_id: int) -> user.User: + """ + Gets a user using this session, connects the session to the User object to allow authenticated actions + + This method ... + 1) gets the username by posting a comment with the user_id as commentee_id. + 2) deletes the posted comment. + 3) fetches other information about the user using Scratch's api.scratch.mit.edu/users/username API. + + Warning: + Every time this functions is run, a comment on your profile is posted and deleted. Therefore, you shouldn't run this too often. + + Args: + user_id (int): User ID of the requested user + + Returns: + scratchattach.user.User: An object that represents the requested user and allows you to perform actions on the user (like user.follow) + """ + return self._make_linked_object("username", self.find_username_from_id(user_id), user.User, exceptions.UserNotFound) + + def connect_project(self, project_id) -> project.Project: + """ + Gets a project using this session, connects the session to the Project object to allow authenticated actions + sess + Args: + project_id (int): ID of the requested project + + Returns: + scratchattach.project.Project: An object that represents the requested project and allows you to perform actions on the project (like project.love) + """ + return self._make_linked_object("id", int(project_id), project.Project, exceptions.ProjectNotFound) + + def connect_studio(self, studio_id) -> studio.Studio: + """ + Gets a studio using this session, connects the session to the Studio object to allow authenticated actions + + Args: + studio_id (int): ID of the requested studio + + Returns: + scratchattach.studio.Studio: An object that represents the requested studio and allows you to perform actions on the studio (like studio.follow) + """ + return self._make_linked_object("id", int(studio_id), studio.Studio, exceptions.StudioNotFound) + + def connect_classroom(self, class_id) -> classroom.Classroom: + """ + Gets a class using this session. + + Args: + class_id (str): class id of the requested class + + Returns: + scratchattach.classroom.Classroom: An object representing the requested classroom + """ + return self._make_linked_object("id", int(class_id), classroom.Classroom, exceptions.ClassroomNotFound) + + def connect_classroom_from_token(self, class_token) -> classroom.Classroom: + """ + Gets a class using this session. + + Args: + class_token (str): class token of the requested class + + Returns: + scratchattach.classroom.Classroom: An object representing the requested classroom + """ + return self._make_linked_object("classtoken", int(class_token), classroom.Classroom, exceptions.ClassroomNotFound) + + def connect_topic(self, topic_id) -> forum.ForumTopic: + """ + Gets a forum topic using this session, connects the session to the ForumTopic object to allow authenticated actions + Data is up-to-date. Data received from Scratch's RSS feed XML API. + + Args: + topic_id (int): ID of the requested forum topic (can be found in the browser URL bar) + + Returns: + scratchattach.forum.ForumTopic: An object that represents the requested forum topic + """ + return self._make_linked_object("id", int(topic_id), forum.ForumTopic, exceptions.ForumContentNotFound) + + def connect_topic_list(self, category_id, *, page=1): + """ + Gets the topics from a forum category. Data web-scraped from Scratch's forums UI. + Data is up-to-date. + + Args: + category_id (str): ID of the forum category + + Keyword Arguments: + page (str): Page of the category topics that should be returned + + Returns: + list: A list containing the forum topics from the specified category + """ + try: + response = requests.get( + f"https://scratch.mit.edu/discuss/{category_id}/?page={page}", headers=self._headers, cookies=self._cookies + ) + soup = BeautifulSoup(response.content, "html.parser") + except Exception as e: + raise exceptions.FetchError(str(e)) + try: + category_name = soup.find("h4").find("span").get_text() + except Exception: + raise exceptions.BadRequest("Invalid category id") + try: + topics = soup.find_all("tr") + topics.pop(0) + return_topics = [] + for topic in topics: + title_link = topic.find("a") + title = title_link.text.strip() + topic_id = title_link["href"].split("/")[-2] + columns = topic.find_all("td") + columns = [column.text for column in columns] + if len(columns) == 1: + continue + last_updated = columns[3].split(" ")[0] + " " + columns[3].split(" ")[1] + return_topics.append( + forum.ForumTopic( + _session=self, + id=int(topic_id), + title=title, + category_name=category_name, + last_updated=last_updated, + reply_count=int(columns[1]), + view_count=int(columns[2]), + ) + ) + return return_topics + except Exception as e: + raise exceptions.ScrapeError(str(e)) + + def connect_featured(self) -> other_apis.FeaturedData: + """ + Request and return connected featured projects and studios from the front page. + """ + return other_apis.get_featured_data(self) + + def connect_message_events(self, *, update_interval=2) -> message_events.MessageEvents: + return message_events.MessageEvents(user.User(username=self.username, _session=self), update_interval=update_interval) + + def connect_filterbot(self, *, log_deletions=True) -> filterbot.Filterbot: + return filterbot.Filterbot(user.User(username=self.username, _session=self), log_deletions=log_deletions) + + def get_session_string(self) -> str: + assert self.session_string + return self.session_string + + def get_headers(self) -> dict[str, str]: + return self._headers + + def get_cookies(self) -> dict[str, str]: + return self._cookies + + +@dataclass +class PreparedSession: + """ + Session that needs to be activated in a context manager first. Do not instantiate this yourself. + """ + + args: Any = field(repr=False) + kwargs: Any = field(repr=False) + _session: Session = field(repr=False, init=False) + + def __enter__(self) -> Session: + self._session = Session(*self.args, **self.kwargs | {"http_session": http._HTTPSession()})._enter() + return self._session + + def __exit__( + self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] + ) -> None: + self._session._exit(exc_type, exc_val, exc_tb) + + +def decode_session_id(session_id: str) -> tuple[dict[str, str], datetime.datetime]: + """ + Extract the JSON data from the main part of a session ID string + Session id is in the format: + :: + + p1 contains a base64 JSON string (if it starts with `.`, then it is zlib compressed) + p2 is a base 62 encoded timestamp + p3 might be a `synchronous signature` for the first 2 parts (might be useless for us) + + The dict has these attributes: + - username + - _auth_user_id + - testcookie + - _auth_user_backend + - token + - login-ip + - _language + - django_timezone + - _auth_user_hash + """ + p1, p2, _ = session_id.split(":") + p1_bytes = base64.urlsafe_b64decode(p1 + "==") + if p1.startswith('".') or p1.startswith("."): + p1_bytes = zlib.decompress(p1_bytes) + return (json.loads(p1_bytes), datetime.datetime.fromtimestamp(commons.b62_decode(p2))) + + +_global_http_session: http._HTTPSession | None = None + + +def _get_global_http_session() -> http._HTTPSession: + global _global_http_session + if _global_http_session is None: + with http._HTTPSession() as session: + _global_http_session = session + return _global_http_session + + +suppressed_login_warning = local() + + +@contextmanager +def suppress_login_warning(): + """ + Suppress the login warning. + """ + suppressed_login_warning.suppressed = getattr(suppressed_login_warning, "suppressed", 0) + try: + suppressed_login_warning.suppressed += 1 + yield + finally: + suppressed_login_warning.suppressed -= 1 + + +def issue_login_warning() -> None: + """ + Issue a login data warning. + """ + if getattr(suppressed_login_warning, "suppressed", 0): + return + warnings.warn( + "IMPORTANT: If you included login credentials directly in your code (e.g. session_id, session_string, ...), then make sure to EITHER instead load them from environment variables or files OR remember to remove them before you share your code with anyone else. If you want to remove this warning, use `warnings.filterwarnings('ignore', category=scratchattach.LoginDataWarning)`", + exceptions.LoginDataWarning, + ) + + +def login_by_id( + session_id: str, *, username: Optional[str] = None, password: Optional[str] = None, xtoken=None +) -> PreparedSession: + """ + Creates a session / log in to the Scratch website with the specified session id. + Structured similarly to Session._connect_object method. + + Args: + session_id (str) + + Keyword arguments: + username (str) + password (str) + xtoken (str) + + Returns: + scratchattach.session.Session: An object that represents the created login / session + """ + issue_login_warning() + if password is not None: + session_data = dict(id=session_id, username=username, password=password) + session_string = base64.b64encode(json.dumps(session_data).encode()).decode() + else: + session_string = None + _session = PreparedSession((), {"id": session_id, "username": username or "", "session_string": session_string}) + return _session + + +def login(username, password, *, timeout: float | int = 10) -> PreparedSession: + """ + Creates a session / log in to the Scratch website with the specified username and password. + + This method ... + 1. creates a session id by posting a login request to Scratch's login API. (If this fails, scratchattach.exceptions.LoginFailure is raised) + 2. fetches the xtoken and other information by posting a request to scratch.mit.edu/session. (If this fails, a warning is displayed) + + Args: + username (str) + password (str) + + Keyword arguments: + timeout (int): Timeout for the request to Scratch's login API (in seconds). Defaults to 10. + + Returns: + scratchattach.session.Session: An object that represents the created login / session + """ + issue_login_warning() + http_session = _get_global_http_session() + _headers = headers.copy() + _headers["Cookie"] = "scratchcsrftoken=a;scratchlanguage=en;" + with http_session.post( + "https://scratch.mit.edu/login/", + shared_http.options().headers(_headers).timeout(timeout).json({"username": username, "password": password}).value, + ) as response: + try: + result = re.search('"(.*)"', response.headers["Set-Cookie"]) + assert result is not None + session_id = str(result.group()) + except Exception: + raise exceptions.LoginFailure( + "Either the provided authentication data is wrong or your network is banned from Scratch.\n\nIf you're using an online IDE (like replit.com) Scratch possibly banned its IP address. In this case, try logging in with your session id: https://github.com/TimMcCool/scratchattach/wiki#logging-in" + ) + with suppress_login_warning(): + return login_by_id(session_id, username=username, password=password) + + +def login_by_session_string(session_string: str) -> PreparedSession: + """ + Login using a session string. + """ + issue_login_warning() + session_string = base64.b64decode(session_string).decode() + session_data = json.loads(session_string) + try: + assert session_data.get("id") + with suppress_login_warning(): + return login_by_id( + session_data["id"], username=session_data.get("username"), password=session_data.get("password") + ) + except Exception: + pass + try: + assert session_data.get("session_id") + with suppress_login_warning(): + return login_by_id( + session_data["session_id"], username=session_data.get("username"), password=session_data.get("password") + ) + except Exception: + pass + try: + assert session_data.get("username") and session_data.get("password") + with suppress_login_warning(): + return login(username=session_data["username"], password=session_data["password"]) + except Exception: + pass + raise ValueError("Couldn't log in.") + + +def login_by_io(file: SupportsRead[str]) -> PreparedSession: + """ + Login using a file object. + """ + with suppress_login_warning(): + return login_by_session_string(file.read()) + + +def login_by_file(file: FileDescriptorOrPath) -> PreparedSession: + """ + Login using a path to a file. + """ + with suppress_login_warning(), open(file, encoding="utf-8") as f: + return login_by_io(f) + + +def login_from_browser(browser: Browser = ANY) -> PreparedSession: + """ + Login from a browser + """ + cookies = cookies_from_browser(browser) + if "scratchsessionsid" in cookies: + with suppress_login_warning(): + return login_by_id(cookies["scratchsessionsid"]) + raise ValueError("Not enough data to log in.") diff --git a/scratchattach/sync_api/site/studio.py b/scratchattach/sync_api/site/studio.py new file mode 100644 index 00000000..160dc0cb --- /dev/null +++ b/scratchattach/sync_api/site/studio.py @@ -0,0 +1,638 @@ +"""Studio class""" + +from __future__ import annotations +import warnings +import json +import random +from dataclasses import dataclass, field +from typing_extensions import Optional +from . import user, comment, project, activity, session +from scratchattach.site.typed_dicts import StudioDict, StudioRoleDict +from ._base import BaseSiteComponent +from scratchattach.utils import exceptions, commons +from scratchattach.utils.commons import api_iterative, headers +from scratchattach.utils.requests import requests + + +@dataclass +class Studio(BaseSiteComponent[StudioDict]): + """ + Represents a Scratch studio. + """ + + id: int + title: Optional[str] = None + description: Optional[str] = None + host_id: Optional[int] = None + "The user id of the studio host" + follower_count: Optional[int] = None + manager_count: Optional[int] = None + project_count: Optional[int] = None + image_url: Optional[str] = None + open_to_all: Optional[bool] = None + "Whether everyone is allowed to add projects" + comments_allowed: Optional[bool] = None + created: Optional[str] = None + modified: Optional[str] = None + _session: Optional[session.Session] = None + + def __post_init__(self): + self.update_function = requests.get + self.update_api = f"https://api.scratch.mit.edu/studios/{self.id}" + if self._session is None: + self._headers = headers + self._cookies = {} + else: + self._headers = self._session._headers + self._cookies = self._session._cookies + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def _update_from_data(self, studio: StudioDict): + self.id = int(studio["id"]) + self.title = studio["title"] + self.description = studio["description"] + self.host_id = studio["host"] + self.open_to_all = studio["open_to_all"] + self.comments_allowed = studio["comments_allowed"] + self.image_url = studio["image"] + self.created = studio["history"]["created"] + self.modified = studio["history"]["modified"] + stats = studio.get("stats", {}) + self.follower_count = stats.get("followers", self.follower_count) + self.manager_count = stats.get("managers", self.manager_count) + self.project_count = stats.get("projects", self.project_count) + return True + + def __str__(self): + ret = f"-S {self.id}" + if self.title: + ret += f" ({self.title})" + return ret + + def __rich__(self): + from rich.panel import Panel + from rich.table import Table + from rich import box + from rich.markup import escape + + url = f"[link={self.url}]{escape(self.title)}[/]" + ret = Table.grid(expand=True) + ret.add_column(ratio=1) + ret.add_column(ratio=3) + info = Table(box=box.SIMPLE) + info.add_column(url, overflow="fold") + info.add_column(f"#{self.id}", overflow="fold") + info.add_row("Host ID", str(self.host_id)) + info.add_row("Followers", str(self.follower_count)) + info.add_row("Projects", str(self.project_count)) + info.add_row("Managers", str(self.manager_count)) + info.add_row("Comments allowed", str(self.comments_allowed)) + info.add_row("Open", str(self.open_to_all)) + info.add_row("Created", self.created) + info.add_row("Modified", self.modified) + desc = Table(box=box.SIMPLE) + desc.add_row("Description", escape(self.description)) + ret.add_row(Panel(info, title=url), Panel(desc, title="Description")) + return ret + + @property + def url(self): + return f"https://scratch.mit.edu/studios/{self.id}" + + @property + def thumbnail(self) -> bytes: + with requests.no_error_handling(): + return requests.get(self.image_url).content + + def follow(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/users/bookmarkers/{self.id}/add/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + + def unfollow(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/users/bookmarkers/{self.id}/remove/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + + def comments(self, *, limit=40, offset=0) -> list[comment.Comment]: + """ + Returns the comments posted on the studio (except for replies. To get replies use :meth:`scratchattach.studio.Studio.get_comment_replies`). + + Keyword Arguments: + page: The page of the comments that should be returned. + limit: Max. amount of returned comments. + + Returns: + list: A list containing the requested comments as Comment objects. + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/studios/{self.id}/comments/", + limit=limit, + offset=offset, + add_params=f"&cachebust={random.randint(0, 9999)}", + ) + for i in response: + i["source"] = "studio" + i["source_id"] = self.id + return commons.parse_object_list(response, comment.Comment, self._session) + + def comment_replies(self, *, comment_id, limit=40, offset=0) -> list[comment.Comment]: + response = commons.api_iterative( + f"https://api.scratch.mit.edu/studios/{self.id}/comments/{comment_id}/replies", + limit=limit, + offset=offset, + add_params=f"&cachebust={random.randint(0, 9999)}", + ) + for x in response: + x["parent_id"] = comment_id + x["source"] = "studio" + x["source_id"] = self.id + return commons.parse_object_list(response, comment.Comment, self._session) + + def comment_by_id(self, comment_id): + r = requests.get(f"https://api.scratch.mit.edu/studios/{self.id}/comments/{comment_id}", timeout=10).json() + if r is None: + raise exceptions.CommentNotFound() + _comment = comment.Comment(id=r["id"], _session=self._session, source=comment.CommentSource.STUDIO, source_id=self.id) + _comment._update_from_data(r) + return _comment + + def post_comment(self, content, *, parent_id="", commentee_id=""): + """ + Posts a comment on the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Args: + content: Content of the comment that should be posted + + Keyword Arguments: + parent_id: ID of the comment you want to reply to. If you don't want to mention a user, don't put the argument. + commentee_id: ID of the user that will be mentioned in your comment and will receive a message about your comment. If you don't want to mention a user, don't put the argument. + + Returns: + scratchattach.comment.Comment: The posted comment as Comment object. + """ + self._assert_auth() + data = {"commentee_id": commentee_id, "content": str(content), "parent_id": parent_id} + headers = dict(self._json_headers) + headers["referer"] = "https://scratch.mit.edu/projects/" + str(self.id) + "/" + r = requests.post( + f"https://api.scratch.mit.edu/proxy/comments/studio/{self.id}/", + headers=headers, + cookies=self._cookies, + data=json.dumps(data), + timeout=10, + ).json() + if "id" not in r: + raise exceptions.CommentPostFailure(r) + _comment = comment.Comment(id=r["id"], _session=self._session, source=comment.CommentSource.STUDIO, source_id=self.id) + _comment._update_from_data(r) + return _comment + + def delete_comment(self, *, comment_id): + """ + Deletes a comment by ID. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + comment_id: The id of the comment that should be deleted + """ + self._assert_auth() + return requests.delete( + f"https://api.scratch.mit.edu/proxy/comments/studio/{self.id}/comment/{comment_id}/", + headers=self._headers, + cookies=self._cookies, + ).headers + + def report_comment(self, *, comment_id): + """ + Reports a comment by ID to the Scratch team. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_project` + + Args: + comment_id: The id of the comment that should be reported + """ + self._assert_auth() + return requests.delete( + f"https://api.scratch.mit.edu/proxy/comments/studio/{self.id}/comment/{comment_id}/report", + headers=self._headers, + cookies=self._cookies, + ) + + def set_thumbnail(self, *, file): + """ + Sets the studio thumbnail. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Keyword Arguments: + file: The path to the image file + + Returns: + str: Scratch cdn link to the set thumbnail + """ + self._assert_auth() + with open(file, "rb") as f: + thumbnail = f.read() + filename = file.replace("\\", "/") + if filename.endswith("/"): + filename = filename[:-1] + filename = filename.split("/").pop() + file_type = filename.split(".").pop() + payload1 = f'------WebKitFormBoundaryhKZwFjoxAyUTMlSh\r\nContent-Disposition: form-data; name="file"; filename="{filename}"\r\nContent-Type: image/{file_type}\r\n\r\n' + payload1 = payload1.encode("utf-8") + payload2 = b"\r\n------WebKitFormBoundaryhKZwFjoxAyUTMlSh--\r\n" + payload = b"".join([payload1, thumbnail, payload2]) + r = requests.post( + f"https://scratch.mit.edu/site-api/galleries/all/{self.id}/", + headers={ + "accept": "*/", + "content-type": "multipart/form-data; boundary=----WebKitFormBoundaryhKZwFjoxAyUTMlSh", + "Referer": "https://scratch.mit.edu/", + "x-csrftoken": "a", + "x-requested-with": "XMLHttpRequest", + }, + data=payload, + cookies=self._cookies, + timeout=10, + ).json() + if "errors" in r: + raise exceptions.BadRequest(", ".join(r["errors"])) + else: + return r["thumbnail_url"] + + def reply_comment(self, content, *, parent_id, commentee_id=""): + """ + Posts a reply to a comment on the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Args: + content: Content of the comment that should be posted + + Warning: + Only replies to top-level comments are shown on the Scratch website. Replies to replies are actually replies to the corresponding top-level comment in the API. + + Therefore, parent_id should be the comment id of a top level comment. + + Keyword Arguments: + parent_id: ID of the comment you want to reply to + commentee_id: ID of the user you are replying to + """ + self._assert_auth() + return self.post_comment(content, parent_id=parent_id, commentee_id=commentee_id) + + def projects(self, limit=40, offset=0) -> list[project.Project]: + """ + Gets the studio projects. + + Keyword arguments: + limit (int): Max amount of returned projects. + offset (int): Offset of the first returned project. + + Returns: + list: A list containing the studio projects as Project objects + """ + response = commons.api_iterative(f"https://api.scratch.mit.edu/studios/{self.id}/projects", limit=limit, offset=offset) + return commons.parse_object_list(response, project.Project, self._session) + + def curators(self, limit=40, offset=0) -> list[user.User]: + """ + Gets the studio curators. + + Keyword arguments: + limit (int): Max amount of returned curators. + offset (int): Offset of the first returned curator. + + Returns: + list: A list containing the studio curators as User objects + """ + response = commons.api_iterative(f"https://api.scratch.mit.edu/studios/{self.id}/curators", limit=limit, offset=offset) + return commons.parse_object_list(response, user.User, self._session, "username") + + def invite_curator(self, curator): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + try: + return requests.put( + f"https://scratch.mit.edu/site-api/users/curators-in/{self.id}/invite_curator/?usernames={curator}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + except Exception: + raise exceptions.Unauthorized + + def promote_curator(self, curator): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + try: + return requests.put( + f"https://scratch.mit.edu/site-api/users/curators-in/{self.id}/promote/?usernames={curator}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + except Exception: + raise exceptions.Unauthorized + + def remove_curator(self, curator): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + try: + return requests.put( + f"https://scratch.mit.edu/site-api/users/curators-in/{self.id}/remove/?usernames={curator}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + except Exception: + raise exceptions.Unauthorized + + def transfer_ownership(self, new_owner, *, password): + """ + Makes another Scratcher studio host. You need to specify your password to do this. + + Arguments: + new_owner (str): Username of new host + + Keyword arguments: + password (str): The password of your Scratch account + + Warning: + This action is irreversible! + """ + self._assert_auth() + try: + return requests.put( + f"https://api.scratch.mit.edu/studios/{self.id}/transfer/{new_owner}", + headers=self._headers, + cookies=self._cookies, + timeout=10, + json={"password": password}, + ).json() + except Exception: + raise exceptions.Unauthorized + + def leave(self): + """ + Removes yourself from the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + return self.remove_curator(self._session._username) + + def add_project(self, project_id): + """ + Adds a project to the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Args: + project_id: Project id of the project that should be added + """ + self._assert_auth() + return requests.post( + f"https://api.scratch.mit.edu/studios/{self.id}/project/{project_id}", headers=self._headers, timeout=10 + ).json() + + def remove_project(self, project_id): + """ + Removes a project from the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + + Args: + project_id: Project id of the project that should be removed + """ + self._assert_auth() + return requests.delete( + f"https://api.scratch.mit.edu/studios/{self.id}/project/{project_id}", headers=self._headers, timeout=10 + ).json() + + def managers(self, limit=40, offset=0): + """ + Gets the studio managers. + + Keyword arguments: + limit (int): Max amount of returned managers + offset (int): Offset of the first returned manager. + + Returns: + list: A list containing the studio managers as user objects + """ + response = commons.api_iterative(f"https://api.scratch.mit.edu/studios/{self.id}/managers", limit=limit, offset=offset) + return commons.parse_object_list(response, user.User, self._session, "username") + + def host(self) -> user.User: + """ + Gets the studio host. + + Returns: + scratchattach.user.User: An object representing the studio host. + """ + managers = self.managers(limit=1, offset=0) + try: + return managers[0] + except Exception: + return None + + def set_fields(self, fields_dict): + """ + Sets fields. Uses the scratch.mit.edu/site-api PUT API. + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/galleries/all/{self.id}/", + headers=headers, + cookies=self._cookies, + data=json.dumps(fields_dict), + timeout=10, + ) + + def set_description(self, new): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self.set_fields({"description": new + "\n"}) + + def set_title(self, new): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self.set_fields({"title": new}) + + def open_projects(self): + """ + Changes the studio settings so everyone (including non-curators) is able to add projects to the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/galleries/{self.id}/mark/open/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + + def close_projects(self): + """ + Changes the studio settings so only curators can add projects to the studio. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/galleries/{self.id}/mark/closed/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + + def turn_off_commenting(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + if self.comments_allowed: + requests.post( + f"https://scratch.mit.edu/site-api/comments/gallery/{self.id}/toggle-comments/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + self.comments_allowed = not self.comments_allowed + + def turn_on_commenting(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + if not self.comments_allowed: + requests.post( + f"https://scratch.mit.edu/site-api/comments/gallery/{self.id}/toggle-comments/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + self.comments_allowed = not self.comments_allowed + + def toggle_commenting(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_studio` + """ + self._assert_auth() + requests.post( + f"https://scratch.mit.edu/site-api/comments/gallery/{self.id}/toggle-comments/", + headers=headers, + cookies=self._cookies, + timeout=10, + ) + self.comments_allowed = not self.comments_allowed + + def activity(self, *, limit=40, offset=0, date_limit=None): + add_params = "" + if date_limit is not None: + add_params = f"&dateLimit={date_limit}" + response = commons.api_iterative( + f"https://api.scratch.mit.edu/studios/{self.id}/activity", limit=limit, offset=offset, add_params=add_params + ) + return commons.parse_object_list(response, activity.Activity, self._session) + + def accept_invite(self): + self._assert_auth() + return requests.put( + f"https://scratch.mit.edu/site-api/users/curators-in/{self.id}/add/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + timeout=10, + ).json() + + def your_role(self) -> StudioRoleDict: + """ + Returns a dict with information about your role in the studio (whether you're following, curating, managing it or are invited) + """ + self._assert_auth() + return requests.get( + f"https://api.scratch.mit.edu/studios/{self.id}/users/{self._session.username}", + headers=self._headers, + cookies=self._cookies, + timeout=10, + ).json() + + def get_exact_project_count(self) -> int: + """ + Get the exact project count of a studio using a binary-search-like strategy + """ + if self.project_count is not None and self.project_count < 100: + return self.project_count + maximum = 100 + minimum = 0 + while True: + if not self.projects(offset=maximum): + break + minimum = maximum + maximum *= 2 + while True: + middle = (minimum + maximum) // 2 + projects = self.projects(limit=40, offset=middle) + if not projects: + maximum = middle + elif len(projects) < 40: + break + else: + minimum = middle + return middle + len(projects) + + +def get_studio(studio_id) -> Studio: + """ + Gets a studio without logging in. + + Args: + studio_id (int): Studio id of the requested studio + + Returns: + scratchattach.studio.Studio: An object representing the requested studio + + Warning: + Any methods that authentication (like studio.follow) will not work on the returned object. + + If you want to use these, get the studio with :meth:`scratchattach.session.Session.connect_studio` instead. + """ + warnings.warn( + "Warning: For methods that require authentication, use session.connect_studio instead of get_studio.\nIf you want to remove this warning, use warnings.filterwarnings('ignore', category=scratchattach.StudioAuthenticationWarning).\nTo ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use `warnings.filterwarnings('ignore', category=scratchattach.GetAuthenticationWarning)`.", + exceptions.StudioAuthenticationWarning, + ) + return commons._get_object("id", studio_id, Studio, exceptions.StudioNotFound) + + +def search_studios(*, query="", mode="trending", language="en", limit=40, offset=0): + if not query: + raise ValueError("The query can't be empty for search") + response = commons.api_iterative( + f"https://api.scratch.mit.edu/search/studios", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}&q={query}", + ) + return commons.parse_object_list(response, Studio) + + +def explore_studios(*, query="", mode="trending", language="en", limit=40, offset=0): + if not query: + raise ValueError("The query can't be empty for explore") + response = commons.api_iterative( + f"https://api.scratch.mit.edu/explore/studios", + limit=limit, + offset=offset, + add_params=f"&language={language}&mode={mode}&q={query}", + ) + return commons.parse_object_list(response, Studio) diff --git a/scratchattach/sync_api/site/typed_dicts.py b/scratchattach/sync_api/site/typed_dicts.py new file mode 100644 index 00000000..e56433b4 --- /dev/null +++ b/scratchattach/sync_api/site/typed_dicts.py @@ -0,0 +1,219 @@ +from __future__ import annotations +from scratchattach.cloud import _base +from typing import TypedDict, Required, NotRequired + + +class SessionUserDict(TypedDict): + id: int + banned: bool + should_vpn: bool + username: str + token: str + thumbnailUrl: str + dateJoined: str + email: str + birthYear: int + birthMonth: int + gender: str + + +class SessionOffenseDict(TypedDict): + expiresAt: float + messageType: str + createdAt: float + + +class EmptySessionOffensesDict(TypedDict): + pass + + +class SessionOffensesDict(TypedDict): + offenses: list[SessionOffenseDict] + showWarning: bool + muteExpiresAt: float + currentMessageType: str + + +class SessionPermissionsDict(TypedDict): + admin: bool + scratcher: bool + new_scratcher: bool + invited_scratcher: bool + social: bool + educator: bool + educator_invitee: bool + student: bool + mute_status: EmptySessionOffensesDict | SessionOffensesDict + + +class SessionFlagsDict(TypedDict): + must_reset_password: bool + must_complete_registration: bool + has_outstanding_email_confirmation: bool + show_welcome: bool + confirm_email_banner: bool + unsupported_browser_banner: bool + with_parent_email: bool + project_comments_enabled: bool + gallery_comments_enabled: bool + userprofile_comments_enabled: bool + everything_is_totally_normal: bool + + +class SessionDict(TypedDict): + user: SessionUserDict + permissions: SessionPermissionsDict + flags: SessionFlagsDict + + +class OcularUserMetaDict(TypedDict): + updated: str + updatedBy: str + + +class OcularUserDict(TypedDict): + _id: str + name: str + status: str + color: str + meta: OcularUserMetaDict + + +class UserHistoryDict(TypedDict): + joined: str + + +UserProfileImagesDict = TypedDict( + "UserProfileImagesDict", {"90x90": str, "60x60": str, "55x55": str, "50x50": str, "32x32": str} +) + + +class UserProfileDict(TypedDict): + id: int + status: str + bio: str + country: str + images: UserProfileImagesDict + membership_label: NotRequired[int] + membership_avatar_badge: NotRequired[int] + + +class UserDict(TypedDict): + id: NotRequired[int] + username: NotRequired[str] + scratchteam: NotRequired[bool] + history: NotRequired[UserHistoryDict] + profile: NotRequired[UserProfileDict] + + +class CloudLogActivityDict(TypedDict): + user: str + verb: str + name: str + variable_name: NotRequired[str] + value: str | float | int + timestamp: int + cloud: _base.AnyCloud + + +class CloudActivityDict(TypedDict): + method: str + name: str + variable_name: NotRequired[str] + value: str | float | int + project_id: int + cloud: _base.AnyCloud + + +class ClassroomDict(TypedDict): + id: int + title: str + description: str + status: str + date_start: NotRequired[str] + date_end: NotRequired[str | None] + images: NotRequired[dict[str, str]] + educator: UserDict + is_closed: NotRequired[bool] + + +class StudioHistoryDict(TypedDict): + created: str + modified: str + + +class StudioStatsDict(TypedDict): + followers: int + managers: int + projects: int + + +class StudioDict(TypedDict): + id: int + title: str + description: str + host: int + open_to_all: bool + comments_allowed: bool + image: str + history: StudioHistoryDict + stats: NotRequired[StudioStatsDict] + + +class StudioRoleDict(TypedDict): + manager: bool + curator: bool + invited: bool + following: bool + + +ProjectImagesDict = TypedDict( + "ProjectImagesDict", {"282x218": str, "216x163": str, "200x200": str, "144x108": str, "135x102": str, "100x80": str} +) + + +class ProjectHistoryDict(TypedDict): + created: str + modified: str + shared: str + + +class ProjectStatsDict(TypedDict): + views: int + loves: int + favorites: int + remixes: int + + +class ProjectRemixDict(TypedDict): + parent: int | None + root: int | None + + +class ProjectDict(TypedDict): + id: int + title: str + description: str + instructions: str + visibility: str + public: bool + comments_allowed: bool + is_published: bool + author: UserDict + image: str + images: ProjectImagesDict + history: ProjectHistoryDict + stats: ProjectStatsDict + remix: ProjectRemixDict + project_token: str + + +class PlaceholderProjectDataMetadataDict(TypedDict): + title: str + description: str + + +class PlaceholderProjectDataDict(TypedDict): + metadata: PlaceholderProjectDataMetadataDict + md5extsToSha256: dict[str, str] + adminOwnershipToken: str | None diff --git a/scratchattach/sync_api/site/user.py b/scratchattach/sync_api/site/user.py new file mode 100644 index 00000000..c21085d2 --- /dev/null +++ b/scratchattach/sync_api/site/user.py @@ -0,0 +1,1018 @@ +"""User class""" + +from __future__ import annotations +import json +import random +import re +import string +import warnings +from typing import Union, cast, Optional, TypedDict +from dataclasses import dataclass, field +from datetime import datetime, timezone +from enum import Enum +from typing_extensions import deprecated +from bs4 import BeautifulSoup, Tag +from ._base import BaseSiteComponent +from scratchattach.eventhandlers import message_events +from scratchattach.utils import commons +from scratchattach.utils import exceptions +from scratchattach.utils.commons import headers +from scratchattach.utils.requests import requests +from . import project +from . import studio +from . import forum +from . import comment +from . import activity +from . import classroom +from . import typed_dicts +from . import session + + +class Rank(Enum): + """ + Possible ranks in scratch + """ + + NEW_SCRATCHER = 0 + SCRATCHER = 1 + SCRATCH_TEAM = 2 + + +class _OcularStatusMeta(TypedDict): + updated: str + updatedBy: str + + +class _OcularStatus(TypedDict): + _id: str + name: str + status: str + color: str + meta: _OcularStatusMeta + + +class Verificator: + def __init__(self, user: User, project_id: int): + self.project = user._make_linked_object("id", project_id, project.Project, exceptions.ProjectNotFound) + self.projecturl = self.project.url + self.code = "".join(random.choices(string.ascii_letters + string.digits, k=8)) + self.username = user.username + + def check(self) -> bool: + return bool( + list( + filter( + lambda x: ( + x.author_name == self.username + and (x.content == self.code or x.content.startswith(self.code) or x.content.endswith(self.code)) + ), + self.project.comments(), + ) + ) + ) + + +@dataclass +class User(BaseSiteComponent[typed_dicts.UserDict]): + """ + Represents a Scratch user. + + Attributes: + + :.join_date: + + :.about_me: + + :.wiwo: Returns the user's 'What I'm working on' section + + :.country: Returns the country from the user profile + + :.icon_url: Returns the link to the user's pfp (90x90) + + :.id: Returns the id of the user + + :.scratchteam: Retuns True if the user is in the Scratch team + + :.update(): Updates the attributes + """ + + username: str = field(kw_only=True, default="") + join_date: str = field(kw_only=True, default="") + about_me: str = field(kw_only=True, default="") + wiwo: str = field(kw_only=True, default="") + country: str = field(kw_only=True, default="") + icon_url: str = field(kw_only=True, default="") + id: int = field(kw_only=True, default=0) + scratchteam: bool = field(kw_only=True, repr=False, default=False) + is_member: bool = field(kw_only=True, repr=False, default=False) + has_ears: bool = field(kw_only=True, repr=False, default=False) + _classroom: tuple[bool, Optional[classroom.Classroom]] = field(init=False, default=(False, None)) + _headers: dict[str, str] = field(init=False, default_factory=headers.copy) + _cookies: dict[str, str] = field(init=False, default_factory=dict) + _json_headers: dict[str, str] = field(init=False, default_factory=dict) + _session: Optional[session.Session] = field(kw_only=True, default=None) + + def __str__(self): + return f"-U {self.username}" + + @property + def status(self) -> str: + return self.wiwo + + @property + def bio(self) -> str: + return self.about_me + + @property + def icon(self) -> bytes: + with requests.no_error_handling(): + return requests.get(self.icon_url).content + + @property + def name(self) -> str: + return self.username + + def __post_init__(self): + self.update_function = requests.get + self.update_api = f"https://api.scratch.mit.edu/users/{self.username}" + if self._session is not None: + self._headers = self._session.get_headers() + self._cookies = self._session.get_cookies() + self._json_headers = dict(self._headers) + self._json_headers["accept"] = "application/json" + self._json_headers["Content-Type"] = "application/json" + + def _update_from_data(self, data: Union[dict, typed_dicts.UserDict]): + data = cast(typed_dicts.UserDict, data) + self.id = data.get("id", self.id) + self.username = data.get("username", self.username) + self.scratchteam = data.get("scratchteam", self.scratchteam) + if history := data.get("history"): + self.join_date = history["joined"] + if profile := data.get("profile"): + self.about_me = profile["bio"] + self.wiwo = profile["status"] + self.country = profile["country"] + self.icon_url = profile["images"]["90x90"] + self.is_member = bool(profile.get("membership_label", False)) + self.has_ears = bool(profile.get("membership_avatar_badge", False)) + return True + + def _assert_permission(self): + self._assert_auth() + if self._session.username != self.username: + raise exceptions.Unauthorized("You need to be authenticated as the profile owner to do this.") + + @property + def url(self): + return f"https://scratch.mit.edu/users/{self.username}" + + def __rich__(self): + from rich.panel import Panel + from rich.table import Table + from rich import box + from rich.markup import escape + + featured_data = self.featured_data() or {} + ocular_data = {} + ocular = "No ocular status" + if status := ocular_data.get("status"): + color_str = "" + color_data = ocular_data.get("color") + if color_data is not None: + color_str = f"[{color_data}] ⬤ [/]" + ocular = f"[i]{escape(status)}[/]{color_str}" + _classroom = self.classroom + url = f"[link={self.url}]{escape(self.username)}[/]" + info = Table(box=box.SIMPLE) + info.add_column(url, overflow="fold") + info.add_column(f"#{self.id}", overflow="fold") + info.add_row("Joined", escape(self.join_date)) + info.add_row("Country", escape(self.country)) + info.add_row("Messages", str(self.message_count())) + info.add_row("Class", str(_classroom.title if _classroom is not None else "None")) + desc = Table("Profile", ocular, box=box.SIMPLE) + desc.add_row("About me", escape(self.about_me)) + desc.add_row("Wiwo", escape(self.wiwo)) + desc.add_row(escape(featured_data.get("label", "Featured Project")), escape(str(self.connect_featured_project()))) + ret = Table.grid(expand=True) + ret.add_column(ratio=1) + ret.add_column(ratio=3) + ret.add_row(Panel(info, title=url), Panel(desc, title="Description")) + return ret + + def connect_featured_project(self) -> Optional[project.Project]: + data = self.featured_data() or {} + if pid := data.get("id"): + return self._session.connect_project(int(pid)) + if projs := self.projects(limit=1): + return projs[0] + return None + + @property + def classroom(self) -> classroom.Classroom | None: + """ + Get a user's associated classroom, and return it as a `scratchattach.classroom.Classroom` object. + If there is no associated classroom, returns `None` + """ + if not self._classroom[0]: + with requests.no_error_handling(): + resp = requests.get(f"https://scratch.mit.edu/users/{self.username}/") + soup = BeautifulSoup(resp.text, "html.parser") + details = soup.find("p", {"class": "profile-details"}) + if details is None: + return None + assert isinstance(details, Tag) + class_name, class_id, is_closed = (None, None, False) + for a in details.find_all("a"): + if not isinstance(a, Tag): + continue + href = str(a.get("href")) + if re.match("/classes/\\d*/", href): + class_name = a.text.strip()[len("Student of: ") :] + is_closed = bool(re.search("\\n *\\(ended\\)", class_name)) + if is_closed: + class_name = re.sub("\\n *\\(ended\\)", "", class_name).strip() + class_id = int(href.split("/")[2]) + break + if class_name: + self._classroom = ( + True, + classroom.Classroom(_session=self._session, id=class_id or 0, title=class_name, is_closed=is_closed), + ) + else: + self._classroom = (True, None) + return self._classroom[1] + + def does_exist(self) -> Optional[bool]: + """ + Returns: + boolean : True if the user exists, False if the user is deleted, None if an error occured + """ + with requests.no_error_handling(): + status_code = requests.get(f"https://scratch.mit.edu/users/{self.username}/").status_code + if status_code == 200: + return True + elif status_code == 404: + return False + return None + + def is_new_scratcher(self): + """ + Returns: + boolean : True if the user has the New Scratcher status, else False + """ + try: + with requests.no_error_handling(): + res = requests.get(f"https://scratch.mit.edu/users/{self.username}/").text + group = res[res.rindex('') :][:70] + return "new scratcher" in group.lower() + except Exception as e: + warnings.warn(f"Caught exception e={e!r}") + return None + + def message_count(self): + return json.loads( + requests.get( + f"https://api.scratch.mit.edu/users/{self.username}/messages/count/?cachebust={random.randint(0, 10000)}", + headers={ + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.3c6 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36" + }, + ).text + )["count"] + + def featured_data(self): + """ + Returns: + dict: Gets info on the user's featured project and featured label (like "Featured project", "My favorite things", etc.) + """ + try: + response = requests.get(f"https://scratch.mit.edu/site-api/users/all/{self.username}/").json() + return { + "label": response["featured_project_label_name"], + "project": dict( + id=str(response["featured_project_data"]["id"]), + author=response["featured_project_data"]["creator"], + thumbnail_url="https://" + response["featured_project_data"]["thumbnail_url"][2:], + title=response["featured_project_data"]["title"], + ), + } + except Exception: + return None + + def unfollowers(self) -> list[User]: + """ + Get all unfollowers by comparing API response and HTML response. + NOTE: This method can take a long time to run. + + Based on https://juegostrower.github.io/unfollowers/ + """ + follower_count = self.follower_count() + usernames = [] + for i in range(1, 2 + follower_count // 60): + with requests.no_error_handling(): + resp = requests.get(f"https://scratch.mit.edu/users/{self.username}/followers/", params={"page": i}) + soup = BeautifulSoup(resp.text, "html.parser") + usernames.extend((span.text.strip() for span in soup.select("span.title"))) + unfollowers = [] + for offset in range(0, follower_count, 40): + unfollowers.extend((user for user in self.followers(offset=offset, limit=40) if user.username not in usernames)) + return unfollowers + + def unfollower_usernames(self) -> list[str]: + return [user.username for user in self.unfollowers()] + + def follower_count(self): + with requests.no_error_handling(): + text = requests.get(f"https://scratch.mit.edu/users/{self.username}/followers/", headers=self._headers).text + return commons.webscrape_count(text, "Followers (", ")") + + def following_count(self): + with requests.no_error_handling(): + text = requests.get(f"https://scratch.mit.edu/users/{self.username}/following/", headers=self._headers).text + return commons.webscrape_count(text, "Following (", ")") + + def followers(self, *, limit=40, offset=0): + """ + Returns: + list: The user's followers as list of scratchattach.user.User objects + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/followers/", limit=limit, offset=offset + ) + return commons.parse_object_list(response, User, self._session, "username") + + def follower_names(self, *, limit=40, offset=0): + """ + Returns: + list: The usernames of the user's followers + """ + return [i.name for i in self.followers(limit=limit, offset=offset)] + + def following(self, *, limit=40, offset=0): + """ + Returns: + list: The users that the user is following as list of scratchattach.user.User objects + """ + response = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/following/", limit=limit, offset=offset + ) + return commons.parse_object_list(response, User, self._session, "username") + + def following_names(self, *, limit=40, offset=0): + """ + Returns: + list: The usernames of the users the user is following + """ + return [i.name for i in self.following(limit=limit, offset=offset)] + + def is_following(self, user: str): + """ + Returns: + boolean: Whether the user is following the user provided as argument + """ + offset = 0 + following = False + while True: + try: + following_names = self.following_names(limit=20, offset=offset) + if user in following_names: + following = True + break + if not following_names: + break + offset += 20 + except Exception as e: + print(f"Warning: API error when performing following check: e={e!r}") + return following + return following + + def is_followed_by(self, user): + """ + Returns: + boolean: Whether the user is followed by the user provided as argument + """ + offset = 0 + followed = False + while True: + try: + followed_names = self.follower_names(limit=20, offset=offset) + if user in followed_names: + followed = True + break + if not followed_names: + break + offset += 20 + except Exception as e: + print(f"Warning: API error when performing following check: e={e!r}") + return followed + return followed + + def is_followed_by_me(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Returns: + boolean: Whether the user is followed by the user currently logged in. + """ + self._assert_auth() + with requests.no_error_handling(): + resp = requests.get( + f"https://scratch.mit.edu/users/{self.username}/", headers=self._headers, cookies=self._cookies + ) + soup = BeautifulSoup(resp.text, "html.parser") + follow_btn = soup.select_one("div.follow-button") + if not follow_btn: + print("Warning: follow button not found in page.") + return False + data_control = follow_btn.get("data-control") + return data_control == "unfollow" + + def project_count(self): + with requests.no_error_handling(): + text = requests.get(f"https://scratch.mit.edu/users/{self.username}/projects/", headers=self._headers).text + return commons.webscrape_count(text, "Shared Projects (", ")") + + def studio_count(self): + with requests.no_error_handling(): + text = requests.get(f"https://scratch.mit.edu/users/{self.username}/studios/", headers=self._headers).text + return commons.webscrape_count(text, "Studios I Curate (", ")") + + def studios_following_count(self): + with requests.no_error_handling(): + text = requests.get( + f"https://scratch.mit.edu/users/{self.username}/studios_following/", headers=self._headers + ).text + return commons.webscrape_count(text, "Studios I Follow (", ")") + + def studios(self, *, limit=40, offset=0) -> list[studio.Studio]: + _studios = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/studios/curate", limit=limit, offset=offset + ) + studios = [] + for studio_dict in _studios: + _studio = studio.Studio(_session=self._session, id=studio_dict["id"]) + _studio._update_from_data(studio_dict) + studios.append(_studio) + return studios + + def studios_following(self) -> list[studio.Studio]: + with requests.no_error_handling(): + resp = requests.get(f"https://scratch.mit.edu/users/{self.username}/studios_following/", headers=self._headers) + soup = BeautifulSoup(resp.text, "html.parser") + grid = soup.select_one(".media-grid") + assert grid is not None + studios: list[studio.Studio] = [] + for studio_elem in grid.select("li.gallery.thumb.item"): + title_span = studio_elem.select_one("span.title") + assert title_span is not None + anchor = title_span.find("a") + assert anchor is not None + href = str(anchor["href"]) + sid = int(href.split("/")[-2]) + title: str = anchor.text + if "\n" in title: + title = title.split("\n")[0] + studios.append(studio.Studio(id=sid, title=title, _session=self._session)) + return studios + + def projects(self, *, limit=40, offset=0) -> list[project.Project]: + """ + Returns: + list: The user's shared projects + """ + _projects = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/projects/", limit=limit, offset=offset, _headers=self._headers + ) + for p in _projects: + p["author"] = {"username": self.username} + return commons.parse_object_list(_projects, project.Project, self._session) + + def loves(self, *, limit=40, offset=0, get_full_project: bool = False) -> list[project.Project]: + """ + Returns: + list: The user's loved projects + """ + if offset < 0: + raise exceptions.BadRequest("offset parameter must be >= 0") + if limit < 0: + raise exceptions.BadRequest("limit parameter must be >= 0") + pages = range(1 + offset // 40, 2 + (offset + limit - 1) // 40) + _projects = [] + for page in pages: + first_idx = (page - 1) * 40 + with requests.no_error_handling(): + page_content = requests.get( + f"https://scratch.mit.edu/projects/all/{self.username}/loves/?page={page}", headers=self._headers + ).content + soup = BeautifulSoup(page_content, "html.parser") + h1_tag = soup.find("h1") + if h1_tag is not None: + if "Whoops! Our server is Scratch'ing its head" in h1_tag.text: + break + for i, project_element in enumerate(soup.find_all("li", {"class": "project thumb item"})): + if offset <= first_idx + i <= offset + limit: + assert isinstance(project_element, Tag) + project_anchors = project_element.find_all("a") + first_anchor = project_anchors[0] + second_anchor = project_anchors[1] + third_anchor = project_anchors[2] + assert isinstance(first_anchor, Tag) + assert isinstance(second_anchor, Tag) + assert isinstance(third_anchor, Tag) + project_id = commons.webscrape_count(first_anchor.attrs["href"], "/projects/", "/") + title = second_anchor.text + author = third_anchor.contents[0] + _project = project.Project( + id=project_id, + _session=self._session, + title=title, + author_name=author, + url=f"https://scratch.mit.edu/projects/{project_id}/", + ) + if get_full_project: + _project.update() + _projects.append(_project) + return _projects + + def loves_count(self): + with requests.no_error_handling(): + text = requests.get(f"https://scratch.mit.edu/projects/all/{self.username}/loves/", headers=self._headers).text + soup = BeautifulSoup(text, "html.parser") + if not soup.find("li", {"class": "project thumb item"}): + return 0 + return commons.webscrape_count(text, "»\n\n (", ")") + + def favorites(self, *, limit=40, offset=0): + """ + Returns: + list: The user's favorite projects + """ + _projects = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/favorites/", limit=limit, offset=offset, _headers=self._headers + ) + return commons.parse_object_list(_projects, project.Project, self._session) + + def favorites_count(self): + with requests.no_error_handling(): + text = requests.get(f"https://scratch.mit.edu/users/{self.username}/favorites/", headers=self._headers).text + return commons.webscrape_count(text, "Favorites (", ")") + + def has_badge(self) -> bool: + """ + Returns: + bool: Whether the user has a scratch membership badge on their profile (located next to the follow button) + """ + with requests.no_error_handling(): + resp = requests.get(self.url) + soup = BeautifulSoup(resp.text, "html.parser") + head = soup.find("div", {"class": "box-head"}) + if not head: + return False + for child in head.children: + if child.name == "img": + if "membership-badge.svg" in child["src"]: + return True + return False + + def toggle_commenting(self): + """ + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_permission() + requests.post( + f"https://scratch.mit.edu/site-api/comments/user/{self.username}/toggle-comments/", + headers=headers, + cookies=self._cookies, + ) + + def viewed_projects(self, limit=24, offset=0): + """ + Returns: + list: The user's recently viewed projects + + You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_permission() + _projects = commons.api_iterative( + f"https://api.scratch.mit.edu/users/{self.username}/projects/recentlyviewed", + limit=limit, + offset=offset, + _headers=self._headers, + ) + return commons.parse_object_list(_projects, project.Project, self._session) + + def set_pfp(self, image: bytes): + """ + Sets the user's profile picture. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + requests.post( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", + headers=self._headers, + cookies=self._cookies, + files={"file": image}, + ) + + def set_bio(self, text): + """ + Sets the user's "About me" section. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + requests.put( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", + headers=self._json_headers, + cookies=self._cookies, + json={"bio": text}, + ) + + def set_wiwo(self, text): + """ + Sets the user's "What I'm working on" section. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + requests.put( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", + headers=self._json_headers, + cookies=self._cookies, + json={"status": text}, + ) + + def set_featured(self, project_id, *, label=""): + """ + Sets the user's featured project. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Args: + project_id: Project id of the project that should be set as featured + + Keyword Args: + label: The label that should appear above the featured project on the user's profile (Like "Featured project", "Featured tutorial", "My favorite things", etc.) + """ + self._assert_permission() + requests.put( + f"https://scratch.mit.edu/site-api/users/all/{self.username}/", + headers=self._json_headers, + cookies=self._cookies, + json={"featured_project": int(project_id), "featured_project_label": label}, + ) + + def set_forum_signature(self, text): + """ + Sets the user's discuss forum signature. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_permission() + headers = { + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "content-type": "application/x-www-form-urlencoded", + "origin": "https://scratch.mit.edu", + "referer": "https://scratch.mit.edu/discuss/settings/TimMcCool/", + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", + } + data = {"csrfmiddlewaretoken": "a", "signature": text, "update": ""} + response = requests.post( + f"https://scratch.mit.edu/discuss/settings/{self.username}/", cookies=self._cookies, headers=headers, data=data + ) + + def post_comment(self, content, *, parent_id="", commentee_id=""): + """ + Posts a comment on the user's profile. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Args: + :param content: Content of the comment that should be posted + + Keyword Arguments: + :param commentee_id: ID of the comment you want to reply to. If you don't want to mention a user, don't put the argument. + :param parent_id: ID of the user that will be mentioned in your comment and will receive a message about your comment. If you don't want to mention a user, don't put the argument. + + Returns: + scratchattach.comment.Comment: An object representing the created comment. + """ + self._assert_auth() + data = {"commentee_id": commentee_id, "content": str(content), "parent_id": parent_id} + r = requests.post( + f"https://scratch.mit.edu/site-api/comments/user/{self.username}/add/", + headers=headers, + cookies=self._cookies, + data=json.dumps(data), + ) + if r.status_code != 200: + if "Looks like we are having issues with our servers!" in r.text: + raise exceptions.BadRequest("Invalid arguments passed") + else: + raise exceptions.CommentPostFailure(r.text) + text = r.text + try: + data = { + "id": text.split('
')[1].split("
")[0].strip(), + "reply_count": 0, + "cached_replies": [], + } + _comment = comment.Comment( + source=comment.CommentSource.USER_PROFILE, + parent_id=None if parent_id == "" else parent_id, + commentee_id=commentee_id, + source_id=self.username, + id=data["id"], + _session=self._session, + datetime=datetime.now(), + ) + _comment._update_from_data(data) + return _comment + except Exception as e: + if '{"error": "isFlood"}' in text: + raise exceptions.CommentPostFailure( + "You are being rate-limited for running this operation too often. Implement a cooldown of about 10 seconds." + ) from e + elif '")[0] + error_data = json.loads(raw_error_data) + expires = error_data["mute_status"]["muteExpiresAt"] + expires = datetime.fromtimestamp(expires, timezone.utc) + raise exceptions.CommentPostFailure(f"You have been muted. Mute expires on {expires}") from e + else: + raise exceptions.FetchError(f"Couldn't parse API response: {r.text!r}") from e + + def reply_comment(self, content, *, parent_id, commentee_id=""): + """ + Replies to a comment given by its id + + Warning: + Only replies to top-level comments are shown on the Scratch website. Replies to replies are actually replies to the corresponding top-level comment in the API. + + Therefore, parent_id should be the comment id of a top level comment. + + Args: + :param content: Content of the comment that should be posted + + Keyword Arguments: + :param parent_id: ID of the comment you want to reply to + :param commentee_id: ID of the user that will be mentioned in your comment and will receive a message about your comment. If you don't want to mention a user, don't put the argument. + """ + return self.post_comment(content, parent_id=parent_id, commentee_id=commentee_id) + + def activity(self, *, limit=1000): + """ + Returns: + list: The user's activity data as parsed list of scratchattach.activity.Activity objects + """ + with requests.no_error_handling(): + soup = BeautifulSoup( + requests.get(f"https://scratch.mit.edu/messages/ajax/user-activity/?user={self.username}&max={limit}").text, + "html.parser", + ) + activities = [] + source = soup.find_all("li") + for data in source: + _activity = activity.Activity(_session=self._session, raw=data) + _activity._update_from_html(data) + activities.append(_activity) + return activities + + def activity_html(self, *, limit=1000): + """ + Returns: + str: The raw user activity HTML data + """ + with requests.no_error_handling(): + return requests.get(f"https://scratch.mit.edu/messages/ajax/user-activity/?user={self.username}&max={limit}").text + + def follow(self): + """ + Follows the user represented by the User object. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/users/followers/{self.username}/add/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + ) + + def unfollow(self): + """ + Unfollows the user represented by the User object. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + """ + self._assert_auth() + requests.put( + f"https://scratch.mit.edu/site-api/users/followers/{self.username}/remove/?usernames={self._session._username}", + headers=headers, + cookies=self._cookies, + ) + + def delete_comment(self, *, comment_id): + """ + Deletes a comment by its ID. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Args: + comment_id: The id of the comment that should be deleted + """ + self._assert_permission() + with requests.no_error_handling(): + return requests.post( + f"https://scratch.mit.edu/site-api/comments/user/{self.username}/del/", + headers=headers, + cookies=self._cookies, + data=json.dumps({"id": str(comment_id)}), + ) + + def report_comment(self, *, comment_id): + """ + Reports a comment by its ID to the Scratch team. You can only use this function if this object was created using :meth:`scratchattach.session.Session.connect_user` + + Args: + comment_id: The id of the comment that should be reported + """ + self._assert_auth() + return requests.post( + f"https://scratch.mit.edu/site-api/comments/user/{self.username}/rep/", + headers=headers, + cookies=self._cookies, + data=json.dumps({"id": str(comment_id)}), + ) + + def comments(self, *, page=1) -> list[comment.Comment] | None: + """ + Returns the comments posted on the user's profile (with replies). + + Keyword Arguments: + page: The page of the comments that should be returned. + + Returns: + list: A list containing the requested comments as Comment objects. + """ + data = [] + with requests.no_error_handling(): + resp = requests.get(f"https://scratch.mit.edu/site-api/comments/user/{self.username}/?page={page}") + if resp.status_code == 404: + return None + soup = BeautifulSoup(resp.content, "html.parser") + _comments = soup.find_all("li", {"class": "top-level-reply"}) + for entity in _comments: + comment_id = entity.find("div", {"class": "comment"})["data-comment-id"] + user = entity.find("a", {"id": "comment-user"})["data-comment-user"] + content = str(entity.find("div", {"class": "content"}).text).strip() + time = entity.find("span", {"class": "time"})["title"] + main_comment = {"id": comment_id, "author": {"username": user}, "content": content, "datetime_created": time} + _comment = comment.Comment( + source=comment.CommentSource.USER_PROFILE, source_id=self.username, _session=self._session + ) + _comment._update_from_data(main_comment) + reply_objs = [] + replies = entity.find_all("li", {"class": "reply"}) + for reply in replies: + r_comment_id = reply.find("div", {"class": "comment"})["data-comment-id"] + r_user = reply.find("a", {"id": "comment-user"})["data-comment-user"] + r_content = ( + str(reply.find("div", {"class": "content"}).text) + .strip() + .replace("\n", "") + .replace(" ", " ") + ) + r_time = reply.find("span", {"class": "time"})["title"] + reply_data = { + "id": r_comment_id, + "author": {"username": r_user}, + "content": r_content, + "datetime_created": r_time, + "parent_id": comment_id, + "cached_parent_comment": _comment, + } + _r_comment = comment.Comment( + source=comment.CommentSource.USER_PROFILE, + source_id=self.username, + _session=self._session, + cached_parent_comment=_comment, + ) + _r_comment._update_from_data(reply_data) + reply_objs.append(_r_comment) + _comment.reply_count = len(reply_objs) + _comment.cached_replies = list(reply_objs) + data.append(_comment) + return data + + def comment_by_id(self, comment_id) -> comment.Comment: + """ + Gets a comment on this user's profile by id. + + Warning: + For comments very far down on the user's profile, this method will take a while to find the comment. Very old comment are deleted from Scratch's database and may not appear. + + Returns: + scratchattach.comments.Comment: The request comment. + """ + page = 1 + page_content = self.comments(page=page) + while page_content != []: + results = list(filter(lambda x: str(x.id) == str(comment_id), page_content)) + if results == []: + results = list( + filter(lambda x: str(x.id) == str(comment_id), [item for x in page_content for item in x.cached_replies]) + ) + if results != []: + return results[0] + else: + return results[0] + page += 1 + page_content = self.comments(page=page) + raise exceptions.CommentNotFound() + + def message_events(self): + return message_events.MessageEvents(self) + + @deprecated("This method is deprecated because ScratchDB is down indefinitely.") + def stats(self): + """ + Gets information about the user's stats. Fetched from ScratchDB. + + Warning: + ScratchDB is down indefinitely, therefore this method is deprecated. + + Returns: + dict: A dict containing the user's stats. If the stats aren't available, all values will be -1. + """ + try: + stats = requests.get(f"https://scratchdb.lefty.one/v3/user/info/{self.username}").json()["statistics"] + stats.pop("ranks") + except Exception: + stats = {"loves": -1, "favorites": -1, "comments": -1, "views": -1, "followers": -1, "following": -1} + return stats + + @deprecated("Warning: ScratchDB is down indefinitely, therefore this method is deprecated.") + def ranks(self): + """ + Gets information about the user's ranks. Fetched from ScratchDB. + + Warning: + ScratchDB is down indefinitely, therefore this method is deprecated. + + Returns: + dict: A dict containing the user's ranks. If the ranks aren't available, all values will be -1. + """ + try: + return requests.get(f"https://scratchdb.lefty.one/v3/user/info/{self.username}").json()["statistics"]["ranks"] + except Exception: + return { + "country": {"loves": 0, "favorites": 0, "comments": 0, "views": 0, "followers": 0, "following": 0}, + "loves": 0, + "favorites": 0, + "comments": 0, + "views": 0, + "followers": 0, + "following": 0, + } + + def ocular_status(self) -> _OcularStatus: + """ + Gets information about the user's ocular status. Ocular is a website developed by jeffalo: https://ocular.jeffalo.net/ + + Returns: + dict + """ + return requests.get(f"https://my-ocular.jeffalo.net/api/user/{self.username}").json() + + def verify_identity(self, *, verification_project_id=395330233): + """ + Can be used in applications to verify a user's identity. + + This function returns a Verifactor object. Attributs of this object: + :.projecturl: The link to the project where the user has to go to verify + :.project: The project where the user has to go to verify as scratchattach.Project object + :.code: The code that the user has to comment + + To check if the user verified successfully, call the .check() function on the returned object. + It will return True if the user commented the code. + """ + v = Verificator(self, verification_project_id) + return v + + def rank(self) -> Rank: + """ + Finds the rank of the user. + Returns a member of the Rank enum: either Rank.NEW_SCRATCHER, Rank.SCRATCHER, or Rank.SCRATCH_TEAM. + May replace user.scratchteam and user.is_new_scratcher in the future. + """ + if self.is_new_scratcher(): + return Rank.NEW_SCRATCHER + if not self.scratchteam: + return Rank.SCRATCHER + return Rank.SCRATCH_TEAM + + +def get_user(username) -> User: + """ + Gets a user without logging in. + + Args: + username (str): Username of the requested user + + Returns: + scratchattach.user.User: An object representing the requested user + + Warning: + Any methods that require authentication (like user.follow) will not work on the returned object. + + If you want to use these, get the user with :meth:`scratchattach.session.Session.connect_user` instead. + """ + warnings.warn( + "Warning: For methods that require authentication, use session.connect_user instead of get_user.\nTo ignore this warning, use warnings.filterwarnings('ignore', category=scratchattach.UserAuthenticationWarning).\nTo ignore all warnings of the type GetAuthenticationWarning, which includes this warning, use `warnings.filterwarnings('ignore', category=scratchattach.GetAuthenticationWarning)`.", + exceptions.UserAuthenticationWarning, + ) + return commons._get_object("username", username, User, exceptions.UserNotFound) diff --git a/scratchattach/utils/exceptions.py b/scratchattach/utils/exceptions.py index 4eb67704..878af889 100644 --- a/scratchattach/utils/exceptions.py +++ b/scratchattach/utils/exceptions.py @@ -1,16 +1,17 @@ # Authentication / Authorization: from __future__ import annotations + class Unauthenticated(Exception): """ Raised when a method that requires a login / session is called on an object that wasn't created with a session. If you create Project, Studio, or User objects using :meth:`scratchattach.get_project`, :meth:`scratchattach.get_studio`, or :meth:`scratchattach.get_user`, they cannot be used for actions that require authentication. Instead, use the following methods to ensure the objects are connected to an authenticated session: - + - :meth:`scratchattach.Session.connect_project` - + - :meth:`scratchattach.Session.connect_user` - + - :meth:`scratchattach.Session.connect_studio` This also applies to cloud variables, forum topics, and forum posts. @@ -30,8 +31,8 @@ class Unauthorized(Exception): def __init__(self, message=""): self.message = ( - f"The user corresponding to the connected login / session is not allowed to perform this action. " - f"{message}") + f"The user corresponding to the connected login / session is not allowed to perform this action. {message}" + ) super().__init__(self.message) @@ -43,8 +44,16 @@ class XTokenError(Exception): """ +class NotATeacherError(Exception): + """ + Raised when the user attempts to do an action which is only associated with teacher accounts, and the + session is not a teacher session. + """ + + # Not found errors: + class UserNotFound(Exception): """ Raised when a non-existent user is requested. @@ -56,6 +65,7 @@ class ProjectNotFound(Exception): Raised when a non-existent project is requested. """ + class ClassroomNotFound(Exception): """ Raised when a non-existent Classroom is requested. @@ -90,8 +100,10 @@ class InvalidTTSGender(Exception): Raised when an invalid TTS gender is provided. """ + # API errors: + class LoginFailure(Exception): """ Raised when the Scratch server doesn't respond with a session id. @@ -111,11 +123,13 @@ class BadRequest(Exception): Raised when the Scratch API responds with a "Bad Request" error message. This can have various reasons. Make sure all provided arguments are valid. """ + class RateLimitedError(Exception): """ Indicates a ratelimit enforced by scratchattach """ + class Response429(Exception): """ Raised when the Scratch API responds with a 429 error. This means that your network was ratelimited or blocked by Scratch. If you're using an online IDE (like replit.com), try running the code on your computer. @@ -140,50 +154,48 @@ class ScrapeError(Exception): """ - # Cloud / encoding errors: + class CloudConnectionError(Exception): """ Raised when connecting to Scratch's cloud server fails. This can have various reasons. """ - class InvalidCloudValue(Exception): """ Raised when a cloud variable is set to an invalid value. """ - class InvalidDecodeInput(Exception): """ Raised when the built-in decoder :meth:`scratchattach.encoder.Encoding.decode` receives an invalid input. """ - # Cloud Requests errors: + class RequestNotFound(Exception): """ Cloud Requests: Raised when a non-existent cloud request is edited using :meth:`scratchattach.cloud_requests.CloudRequests.edit_request`. """ - # Websocket server errors: + class WebsocketServerError(Exception): """ Raised when the self-hosted cloud websocket server fails to start. """ - # Editor errors: + class UnclosedJSONError(Exception): """ Raised when a JSON string is never closed. @@ -225,52 +237,62 @@ class BadScript(Exception): Raised when the block script cannot allow for the operation """ + # Warnings + class LoginDataWarning(UserWarning): """ Warns you not to accidentally share your login data. """ + class InvalidUpdateWarning(UserWarning): """ Warns you that something cannot be updated. """ - + + class GetAuthenticationWarning(UserWarning): """ All authentication warnings. """ - + + class UserAuthenticationWarning(GetAuthenticationWarning): """ Warns you to use session.connect_user instead of user.get_user for actions that require authentication. """ + class ProjectAuthenticationWarning(GetAuthenticationWarning): """ - Warns you to use session.connect_project instead of project.get_project + Warns you to use session.connect_project instead of project.get_project for actions that require authentication. """ + class StudioAuthenticationWarning(GetAuthenticationWarning): """ - Warns you to use session.connect_studio instead of studio.get_studio + Warns you to use session.connect_studio instead of studio.get_studio for actions that require authentication. """ + class ClassroomAuthenticationWarning(GetAuthenticationWarning): """ - Warns you to use session.connect_classroom or session.connect_classroom_from_token instead of classroom.get_classroom + Warns you to use session.connect_classroom or session.connect_classroom_from_token instead of classroom.get_classroom for actions that require authentication. """ + class CloudAuthenticationWarning(GetAuthenticationWarning): """ - Warns you about usage of + Warns you about usage of """ + class UnexpectedWebsocketEventWarning(RuntimeWarning): """ Warns about an unexpected occurrence with a websocket. diff --git a/scratchattach/utils/requests.py b/scratchattach/utils/requests.py index f343cd4c..c8c789d4 100644 --- a/scratchattach/utils/requests.py +++ b/scratchattach/utils/requests.py @@ -9,17 +9,37 @@ from dataclasses import dataclass, field import json -from aiohttp.cookiejar import DummyCookieJar +from aiohttp.cookiejar import DummyCookieJar as DummyCookieJar2 from typing_extensions import override from requests import Session as HTTPSession from requests import Response +from requests.cookies import RequestsCookieJar import aiohttp from . import exceptions from . import optional_async + +class DummyCookieJar(RequestsCookieJar): + def set_cookie(self, *args, **kwargs): + pass + + def update(self, *args, **kwargs): + pass + + def copy(self): + return DummyCookieJar() + + def __setitem__(self, name, value): + pass + + def set(self, *args, **kwargs): + pass + + proxies: Optional[MutableMapping[str, str]] = None + class HTTPMethod(Enum): GET = auto() POST = auto() @@ -29,6 +49,7 @@ class HTTPMethod(Enum): OPTIONS = auto() PATCH = auto() TRACE = auto() + @classmethod def of(cls, name: str) -> HTTPMethod: member_map = { @@ -39,20 +60,22 @@ def of(cls, name: str) -> HTTPMethod: "HEAD": cls.HEAD, "OPTIONS": cls.OPTIONS, "PATCH": cls.PATCH, - "TRACE": cls.TRACE + "TRACE": cls.TRACE, } return member_map[name] + class AnyHTTPResponse(ABC): request_method: HTTPMethod status_code: int content: bytes text: str headers: dict[str, str] - + def json(self) -> Any: return json.loads(self.text) + @dataclass class HTTPResponse(AnyHTTPResponse): request_method: HTTPMethod = field(kw_only=True) @@ -61,8 +84,10 @@ class HTTPResponse(AnyHTTPResponse): text: str = field(kw_only=True) headers: dict[str, str] = field(kw_only=True) + class OAHTTPSession(ABC): error_handling: bool = True + @abstractmethod def sync_request( self, @@ -73,10 +98,10 @@ def sync_request( headers: Optional[dict[str, str]] = None, params: Optional[dict[str, str]] = None, data: Optional[Union[dict[str, str], str]] = None, - json: Optional[Any] = None + json: Optional[Any] = None, ) -> AnyHTTPResponse: pass - + @abstractmethod async def async_request( self, @@ -87,7 +112,7 @@ async def async_request( headers: Optional[dict[str, str]] = None, params: Optional[dict[str, str]] = None, data: Optional[Union[dict[str, str], str]] = None, - json: Optional[Any] = None + json: Optional[Any] = None, ) -> AnyHTTPResponse: pass @@ -98,10 +123,9 @@ def check_response(self, r: AnyHTTPResponse): raise exceptions.APIError("Internal Scratch server error") if r.status_code == 429: raise exceptions.Response429("You are being rate-limited (or blocked) by Scratch") - if r.json() == {"code":"BadRequest","message":""}: + if r.json() == {"code": "BadRequest", "message": ""}: raise exceptions.BadRequest("Make sure all provided arguments are valid") - - + def request( self, method: Union[HTTPMethod, str], @@ -111,21 +135,14 @@ def request( headers: Optional[dict[str, str]] = None, params: Optional[dict[str, str]] = None, data: Optional[Union[dict[str, str], str]] = None, - json: Optional[Any] = None + json: Optional[Any] = None, ) -> optional_async.CARequest: if isinstance(method, str): method = HTTPMethod.of(method.upper()) return optional_async.CARequest( - self, - method, - url, - cookies = cookies, - headers = headers, - params = params, - data = data, - json = json + self, method, url, cookies=cookies, headers=headers, params=params, data=data, json=json ) - + @contextmanager def no_error_handling(self) -> Iterator[None]: val_before = self.error_handling @@ -134,7 +151,7 @@ def no_error_handling(self) -> Iterator[None]: yield finally: self.error_handling = val_before - + @contextmanager def yes_error_handling(self) -> Iterator[None]: val_before = self.error_handling @@ -144,19 +161,22 @@ def yes_error_handling(self) -> Iterator[None]: finally: self.error_handling = val_before + class SyncRequests(OAHTTPSession): @override - def sync_request(self, method, url, *, cookies = None, headers = None, params = None, data = None, json = None): + def sync_request( + self, method, url, *, cookies=None, headers=None, params=None, data=None, json=None + ): try: r = requests.request( method.name, url, - cookies = cookies, - headers = headers, - params = params, - data = data, - json = json, - proxies = proxies + cookies=cookies, + headers=headers, + params=params, + data=data, + json=json, + proxies=proxies, ) except Exception as e: raise exceptions.FetchError(e) @@ -165,34 +185,42 @@ def sync_request(self, method, url, *, cookies = None, headers = None, params = status_code=r.status_code, content=r.content, text=r.text, - headers=r.headers + headers=r.headers, ) if self.error_handling: self.check_response(response) return response - - async def async_request(self, method, url, *, cookies = None, headers = None, params = None, data = None, json = None): + + async def async_request( + self, method, url, *, cookies=None, headers=None, params=None, data=None, json=None + ): raise NotImplementedError() + class AsyncRequests(OAHTTPSession): client_session: aiohttp.ClientSession + async def __aenter__(self) -> Self: - self.client_session = await aiohttp.ClientSession(cookie_jar=DummyCookieJar()).__aenter__() + self.client_session = await aiohttp.ClientSession(cookie_jar=DummyCookieJar2()).__aenter__() return self - + async def __aexit__( self, exc_type: Optional[type[BaseException]] = None, exc_val: Optional[BaseException] = None, - exc_tb: Optional[TracebackType] = None + exc_tb: Optional[TracebackType] = None, ) -> None: await self.client_session.__aexit__(exc_type, exc_val, exc_tb) - + @override - def sync_request(self, method, url, *, cookies = None, headers = None, params = None, data = None, json = None): + def sync_request( + self, method, url, *, cookies=None, headers=None, params=None, data=None, json=None + ): raise NotImplementedError() - - async def async_request(self, method, url, *, cookies = None, headers = None, params = None, data = None, json = None): + + async def async_request( + self, method, url, *, cookies=None, headers=None, params=None, data=None, json=None + ): proxy = None if url.startswith("http"): proxy = proxies.get("http") @@ -201,12 +229,12 @@ async def async_request(self, method, url, *, cookies = None, headers = None, pa async with self.client_session.request( method.name, url, - cookies = cookies, - headers = headers, - params = params, - data = data, - json = json, - proxy = proxy + cookies=cookies, + headers=headers, + params=params, + data=data, + json=json, + proxy=proxy, ) as resp: assert isinstance(resp, aiohttp.ClientResponse) content = await resp.read() @@ -219,18 +247,24 @@ async def async_request(self, method, url, *, cookies = None, headers = None, pa status_code=resp.status, content=content, text=text, - headers=resp.headers + headers=resp.headers, ) if self.error_handling: self.check_response(response) return response + class Requests(HTTPSession): """ Centralized HTTP request handler (for better error handling and proxies) """ + error_handling: bool = True + def __init__(self): + super().__init__() + self.cookies = DummyCookieJar() + def check_response(self, r: Response): if r.status_code == 403 or r.status_code == 401: raise exceptions.Unauthorized(f"Request content: {r.content!r}") @@ -238,7 +272,7 @@ def check_response(self, r: Response): raise exceptions.APIError("Internal Scratch server error") if r.status_code == 429: raise exceptions.Response429("You are being rate-limited (or blocked) by Scratch") - if r.json() == {"code":"BadRequest","message":""}: + if r.json() == {"code": "BadRequest", "message": ""}: raise exceptions.BadRequest("Make sure all provided arguments are valid") @override @@ -284,7 +318,7 @@ def put(self, *args, **kwargs): if self.error_handling: self.check_response(r) return r - + @contextmanager def no_error_handling(self) -> Iterator[None]: val_before = self.error_handling @@ -293,7 +327,7 @@ def no_error_handling(self) -> Iterator[None]: yield finally: self.error_handling = val_before - + @contextmanager def yes_error_handling(self) -> Iterator[None]: val_before = self.error_handling @@ -303,4 +337,5 @@ def yes_error_handling(self) -> Iterator[None]: finally: self.error_handling = val_before + requests = Requests() diff --git a/tests/test_memberships.py b/tests/test_memberships.py index 72dc5fc9..f974cbc4 100644 --- a/tests/test_memberships.py +++ b/tests/test_memberships.py @@ -11,7 +11,7 @@ def test_memberships(): assert not u1.has_ears assert u1.has_badge() - u2 = sa.get_user("ceebee") + u2 = sa.get_user("scratchteam") assert u2.is_member assert u2.has_ears assert u2.has_badge() diff --git a/tests/uv.lock b/tests/uv.lock index 778c3dfa..efb52b88 100644 --- a/tests/uv.lock +++ b/tests/uv.lock @@ -109,6 +109,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -448,6 +461,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + [[package]] name = "idna" version = "3.13" @@ -887,6 +937,7 @@ dependencies = [ { name = "aiohttp" }, { name = "browser-cookie3" }, { name = "bs4" }, + { name = "httpx" }, { name = "requests" }, { name = "rich" }, { name = "simplewebsocketserver" }, @@ -899,6 +950,7 @@ requires-dist = [ { name = "aiohttp" }, { name = "browser-cookie3" }, { name = "bs4" }, + { name = "httpx", specifier = ">=0.28.1" }, { name = "lark", marker = "extra == 'lark'" }, { name = "requests" }, { name = "rich" }, diff --git a/uv.lock b/uv.lock index 6dae1154..5cc488cb 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,12 @@ version = 1 revision = 3 -requires-python = ">=3.12" +requires-python = ">=3.12.12" + +[manifest] +members = [ + "codegen", + "scratchattach", +] [[package]] name = "aiohappyeyeballs" @@ -109,6 +115,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, +] + [[package]] name = "attrs" version = "25.4.0" @@ -224,6 +243,17 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, ] +[[package]] +name = "codegen" +version = "0.1.0" +source = { virtual = "codegen" } +dependencies = [ + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [{ name = "ruff" }] + [[package]] name = "frozenlist" version = "1.8.0" @@ -313,6 +343,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -782,6 +849,7 @@ dependencies = [ { name = "aiohttp" }, { name = "browser-cookie3" }, { name = "bs4" }, + { name = "httpx" }, { name = "requests" }, { name = "rich" }, { name = "simplewebsocketserver" }, @@ -807,6 +875,7 @@ requires-dist = [ { name = "aiohttp" }, { name = "browser-cookie3" }, { name = "bs4" }, + { name = "httpx", specifier = ">=0.28.1" }, { name = "lark", marker = "extra == 'lark'" }, { name = "requests" }, { name = "rich" },