diff --git a/README.md b/README.md index 2cb9a56..cd8fae5 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ has some usefull stuff as well ## Installation -You can install `herolib` directly from the Git repository using `uv pip` (or `pip`): +You can install `herolib` directly from the Git repository using `uv pip`: ```bash uv pip install git+https://git.ourworld.tf/herocode/herolib_python.git @@ -27,6 +27,10 @@ import herolib.core.loghandler.mylogging from herolib.core.loghandler.mylogging import MyLogger ``` +## how to integrate python in other projects + +see [Python Herolib Integration](pythonsetup/README.md) + ## Version Control This library follows standard Git version control practices. Releases can be managed by tagging specific commits in the Git repository. Users installing directly from the Git URL can specify a particular branch, tag, or commit hash to get a specific version. For example: diff --git a/herolib/core/logger/__pycache__/__init__.cpython-313.pyc b/herolib/core/logger/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..c7e1fb1 Binary files /dev/null and b/herolib/core/logger/__pycache__/__init__.cpython-313.pyc differ diff --git a/herolib/core/logger/__pycache__/factory.cpython-313.pyc b/herolib/core/logger/__pycache__/factory.cpython-313.pyc new file mode 100644 index 0000000..907292a Binary files /dev/null and b/herolib/core/logger/__pycache__/factory.cpython-313.pyc differ diff --git a/herolib/core/logger/__pycache__/model.cpython-313.pyc b/herolib/core/logger/__pycache__/model.cpython-313.pyc new file mode 100644 index 0000000..2182e87 Binary files /dev/null and b/herolib/core/logger/__pycache__/model.cpython-313.pyc differ diff --git a/herolib/core/logger/__pycache__/search.cpython-313.pyc b/herolib/core/logger/__pycache__/search.cpython-313.pyc new file mode 100644 index 0000000..5ae3dfa Binary files /dev/null and b/herolib/core/logger/__pycache__/search.cpython-313.pyc differ diff --git a/herolib/core/logger/log_test.py b/herolib/core/logger/log_test.py index 278b3a9..ce4b5fb 100644 --- a/herolib/core/logger/log_test.py +++ b/herolib/core/logger/log_test.py @@ -1,10 +1,11 @@ import unittest import os import shutil -from lib.core.logger.factory import new -from lib.core.logger.model import LogItemArgs, LogType, Logger # Import Logger class -from lib.data.ourtime.ourtime import new as ourtime_new, now as ourtime_now -from lib.core.pathlib.pathlib import get_file, ls, rmdir_all +from herolib.core.logger.factory import new +from herolib.core.logger.model import LogItemArgs, LogType, Logger # Import Logger class +from herolib.data.ourtime.ourtime import new as ourtime_new, now as ourtime_now +from herolib.core.pathlib.pathlib import get_file, ls, rmdir_all +from herolib.core.logger.search import search, SearchArgs class TestLogger(unittest.TestCase): def setUp(self): @@ -85,18 +86,18 @@ class TestLogger(unittest.TestCase): self.assertEqual(len(files), 2) # Expecting two files: 2022-12-05-20.log and 2022-12-05-22.log # Test search functionality - items_stdout = logger.search( + items_stdout = search(logger, SearchArgs( timestamp_from=ourtime_new('2022-11-01 20:14:35'), timestamp_to=ourtime_new('2025-11-01 20:14:35'), logtype=LogType.STDOUT - ) + )) self.assertEqual(len(items_stdout), 2) - items_error = logger.search( + items_error = search(logger, SearchArgs( timestamp_from=ourtime_new('2022-11-01 20:14:35'), timestamp_to=ourtime_new('2025-11-01 20:14:35'), logtype=LogType.ERROR - ) + )) self.assertEqual(len(items_error), 4) # Test specific log content @@ -115,34 +116,34 @@ class TestLogger(unittest.TestCase): self.assertTrue(found_stdout_log, "Expected stdout log content not found") # Test search by category - items_test_app = logger.search( + items_test_app = search(logger, SearchArgs( timestamp_from=ourtime_new('2022-11-01 20:14:35'), timestamp_to=ourtime_new('2025-11-01 20:14:35'), cat='test-app' - ) + )) self.assertEqual(len(items_test_app), 2) - items_error_test = logger.search( + items_error_test = search(logger, SearchArgs( timestamp_from=ourtime_new('2022-11-01 20:14:35'), timestamp_to=ourtime_new('2025-11-01 20:14:35'), cat='error-test' - ) + )) self.assertEqual(len(items_error_test), 4) # Test search by log content - items_with_aaa = logger.search( + items_with_aaa = search(logger, SearchArgs( timestamp_from=ourtime_new('2022-11-01 20:14:35'), timestamp_to=ourtime_new('2025-11-01 20:14:35'), log='aaa' - ) + )) self.assertEqual(len(items_with_aaa), 2) # Test search with timestamp range - items_specific_time = logger.search( + items_specific_time = search(logger, SearchArgs( timestamp_from=ourtime_new('2022-12-05 22:00:00'), timestamp_to=ourtime_new('2022-12-05 23:00:00'), logtype=LogType.ERROR - ) + )) self.assertEqual(len(items_specific_time), 2) diff --git a/herolib/core/logger/search.py b/herolib/core/logger/search.py index 4ac3e81..fe9b164 100644 --- a/herolib/core/logger/search.py +++ b/herolib/core/logger/search.py @@ -16,122 +16,98 @@ class SearchArgs: self.logtype = logtype self.maxitems = maxitems -def process(result: List[LogItem], current_item: LogItem, current_time: OurTime, - args: SearchArgs, from_time: int, to_time: int): - # Add previous item if it matches filters - log_epoch = current_item.timestamp.unix() - if log_epoch < from_time or log_epoch > to_time: - return - - cat_match = (args.cat == '' or current_item.cat.strip() == args.cat) - log_match = (args.log == '' or args.log.lower() in current_item.log.lower()) - logtype_match = (args.logtype is None or current_item.logtype == args.logtype) - - if cat_match and log_match and logtype_match: - result.append(current_item) - def search(l: Logger, args_: SearchArgs) -> List[LogItem]: args = args_ - # Format category (max 10 chars, ascii only) args.cat = name_fix(args.cat) if len(args.cat) > 10: raise ValueError('category cannot be longer than 10 chars') - timestamp_from = args.timestamp_from if args.timestamp_from else OurTime() - timestamp_to = args.timestamp_to if args.timestamp_to else OurTime() + from_time = args.timestamp_from.unix() if args.timestamp_from else 0 + to_time = args.timestamp_to.unix() if args.timestamp_to else ourtime_new('2100-01-01').unix() - # Get time range - from_time = timestamp_from.unix() - to_time = timestamp_to.unix() if from_time > to_time: - raise ValueError(f'from_time cannot be after to_time: {from_time} < {to_time}') - + raise ValueError(f'from_time cannot be after to_time: {from_time} > {to_time}') + result: List[LogItem] = [] - - # Find log files in time range + if not os.path.exists(l.path.path): + return [] files = sorted(os.listdir(l.path.path)) for file in files: if not file.endswith('.log'): continue - # Parse dayhour from filename - dayhour = file[:-4] # remove .log + dayhour = file[:-4] try: file_time = ourtime_new(dayhour) except ValueError: - continue # Skip if filename is not a valid time format - - current_time = OurTime() - current_item = LogItem(OurTime(), "", "", LogType.STDOUT) # Initialize with dummy values - collecting = False - - # Skip if file is outside time range - if file_time.unix() < from_time or file_time.unix() > to_time: continue - # Read and parse log file - content = "" + file_hour_start_unix = file_time.unix() + file_hour_end_unix = file_hour_start_unix + 3599 + if file_hour_end_unix < from_time or file_hour_start_unix > to_time: + continue + try: with open(os.path.join(l.path.path, file), 'r') as f: content = f.read() except FileNotFoundError: continue - lines = content.split('\n') + current_time = None + current_item = None - for line in lines: + for line in content.splitlines(): if len(result) >= args.maxitems: - return result + break - line_trim = line.strip() - if not line_trim: + if not line.strip() and current_item: + if from_time <= current_item.timestamp.unix() <= to_time: + if (not args.cat or args.cat == current_item.cat) and \ + (not args.log or args.log.lower() in current_item.log.lower()) and \ + (args.logtype is None or args.logtype == current_item.logtype): + result.append(current_item) + current_item = None continue - # Check if this is a timestamp line - if not (line.startswith(' ') or line.startswith('E')): + if not line.startswith(' ') and not line.startswith('E'): + if current_item: + if from_time <= current_item.timestamp.unix() <= to_time: + if (not args.cat or args.cat == current_item.cat) and \ + (not args.log or args.log.lower() in current_item.log.lower()) and \ + (args.logtype is None or args.logtype == current_item.logtype): + result.append(current_item) try: - current_time = ourtime_new(line_trim) + current_time = ourtime_new(f"{file_time.day()} {line.strip()}") + current_item = None except ValueError: - continue # Skip if not a valid timestamp line + current_time = None + current_item = None + elif current_time: + if line.startswith(' ') or line.startswith('E'): + if len(line) > 14 and line[13] == '-': + if current_item: + if from_time <= current_item.timestamp.unix() <= to_time: + if (not args.cat or args.cat == current_item.cat) and \ + (not args.log or args.log.lower() in current_item.log.lower()) and \ + (args.logtype is None or args.logtype == current_item.logtype): + result.append(current_item) + + is_error = line.startswith('E') + logtype = LogType.ERROR if is_error else LogType.STDOUT + cat = line[2:12].strip() + log_content = line[15:] - if collecting: - process(result, current_item, current_time, args, from_time, to_time) - collecting = False - continue - - if collecting and len(line) > 14 and line[13] == '-': - process(result, current_item, current_time, args, from_time, to_time) - collecting = False - - # Parse log line - is_error = line.startswith('E') - if not collecting: - # Start new item - cat_start = 2 - cat_end = 12 - log_start = 15 - - if len(line) < log_start: - continue # Line too short to contain log content - - current_item = LogItem( - timestamp=current_time, - cat=line[cat_start:cat_end].strip(), - log=line[log_start:].strip(), - logtype=LogType.ERROR if is_error else LogType.STDOUT - ) - collecting = True - else: - # Continuation line - if len(line_trim) < 16: # Check for minimum length for continuation line - current_item.log += '\n' + line_trim - else: - current_item.log += '\n' + line[15:].strip() # Use strip for continuation lines - - # Add last item if collecting - if collecting: - process(result, current_item, current_time, args, from_time, to_time) + current_item = LogItem(timestamp=current_time, cat=cat, log=log_content.strip(), logtype=logtype) + elif current_item: + current_item.log += "\n" + (line[15:] if len(line) >15 else line) + + if current_item: + if from_time <= current_item.timestamp.unix() <= to_time: + if (not args.cat or args.cat == current_item.cat) and \ + (not args.log or args.log.lower() in current_item.log.lower()) and \ + (args.logtype is None or args.logtype == current_item.logtype): + result.append(current_item) return result \ No newline at end of file diff --git a/herolib/core/pathlib/__pycache__/__init__.cpython-313.pyc b/herolib/core/pathlib/__pycache__/__init__.cpython-313.pyc index 4123a7b..66f38af 100644 Binary files a/herolib/core/pathlib/__pycache__/__init__.cpython-313.pyc and b/herolib/core/pathlib/__pycache__/__init__.cpython-313.pyc differ diff --git a/herolib/core/texttools/__pycache__/__init__.cpython-313.pyc b/herolib/core/texttools/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..bd72704 Binary files /dev/null and b/herolib/core/texttools/__pycache__/__init__.cpython-313.pyc differ diff --git a/herolib/core/texttools/__pycache__/texttools.cpython-313.pyc b/herolib/core/texttools/__pycache__/texttools.cpython-313.pyc new file mode 100644 index 0000000..8a5c11f Binary files /dev/null and b/herolib/core/texttools/__pycache__/texttools.cpython-313.pyc differ diff --git a/herolib/core/texttools/texttools.py b/herolib/core/texttools/texttools.py index 5a60056..1a37826 100644 --- a/herolib/core/texttools/texttools.py +++ b/herolib/core/texttools/texttools.py @@ -1,4 +1,6 @@ import re +from datetime import datetime +import os def name_fix(name: str) -> str: # VLang's name_fix converts '-' to '_' and cleans up special chars. diff --git a/herolib/data/__pycache__/__init__.cpython-313.pyc b/herolib/data/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..3a0f778 Binary files /dev/null and b/herolib/data/__pycache__/__init__.cpython-313.pyc differ diff --git a/herolib/data/ourtime/__pycache__/__init__.cpython-313.pyc b/herolib/data/ourtime/__pycache__/__init__.cpython-313.pyc index b15cd97..1121c51 100644 Binary files a/herolib/data/ourtime/__pycache__/__init__.cpython-313.pyc and b/herolib/data/ourtime/__pycache__/__init__.cpython-313.pyc differ diff --git a/pythonsetup/README.md b/pythonsetup/README.md new file mode 100644 index 0000000..f460beb --- /dev/null +++ b/pythonsetup/README.md @@ -0,0 +1,60 @@ +# Python Example setup + +This repository contains the setup scripts and configuration files for how we use python in threefold/ourworld + +```bash +./install.sh +``` + +This script will: + +1. Check if `uv` is installed (a fast Python package installer and resolver) +2. Install `uv` if it's not found +3. Initialize a uv project if `pyproject.toml` doesn't exist +4. Sync all project dependencies using `uv sync` +5. Install the herolib package from git repository +6. Create necessary directories: (remove if you don't need this for your project) + - `static/css`, `static/js`, `static/images` for static assets + - `templates` for HTML templates + - `md` for markdown files + +check how we install herolib here + - `pip install git+https://github.com/ThreeFoldTech/herolib.git` and also have support for the local checked out version + +## Running the Server + +### Production Mode + +To start the server in production mode, run: + +```bash +./start_server.sh +``` + +This script will: +1. Set environment variables for production +2. Check and free port 9922 if it's in use +3. Start the web server on port 9922 +4. Make the server available at http://localhost:9922 + +### Development/Debug Mode + +To start the server in development/debug mode, run: + +```bash +./start_server_debug.sh +``` + +This script will: +1. Set environment variables for development +2. Enable debug mode +3. Check and free port 9922 if it's in use +4. Start the web server on port 9922 with debug options +5. Make the server available at http://localhost:9922 + +## Environment Setup + +The `pipenv.sh` script is automatically sourced by the startup scripts and handles: +1. Setting the PYTHONPATH to include the src directory +2. Creating a virtual environment with uv if it doesn't exist +3. Activating the virtual environment diff --git a/pythonsetup/install.sh b/pythonsetup/install.sh new file mode 100755 index 0000000..c50ff41 --- /dev/null +++ b/pythonsetup/install.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# KnowledgeCenter Web Server Installation Script +# This script sets up the necessary environment for the Flask web server. + +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +echo -e "${BLUE}🔧 Setting up KnowledgeCenter Web Server Environment${NC}" +echo "==================================================" + +# Check if uv is installed +if ! command -v uv &> /dev/null; then + echo -e "${YELLOW}⚠️ uv is not installed. Installing uv...${NC}" + curl -LsSf https://astral.sh/uv/install.sh | sh + source $HOME/.cargo/env + echo -e "${GREEN}✅ uv installed${NC}" +fi + +echo -e "${GREEN}✅ uv found${NC}" + +# Initialize uv project if not already done +if [ ! -f "pyproject.toml" ]; then + echo -e "${YELLOW}⚠️ No pyproject.toml found. Initializing uv project...${NC}" + uv init --no-readme --python 3.13 + echo -e "${GREEN}✅ uv project initialized${NC}" +fi + +# Sync dependencies +echo -e "${YELLOW}📦 Installing dependencies with uv...${NC}" +uv sync +if [ -d "$HOME/code/git.ourworld.tf/herocode/herolib_python/herolib" ]; then + echo -e "${GREEN}✅ Found local herolib, installing...${NC}" + uv pip install -e "$HOME/code/git.ourworld.tf/herocode/herolib_python" +else + echo -e "${YELLOW}📦 Local herolib not found, installing from git...${NC}" + uv pip install herolib@git+https://git.ourworld.tf/herocode/herolib_python.git --force-reinstall --no-cache-dir +fi +echo -e "${GREEN}✅ Dependencies installed${NC}" + +# Create necessary directories +mkdir -p static/css static/js static/images +mkdir -p templates +mkdir -p md + +echo -e "${GREEN}✅ Directory structure verified${NC}" + +echo -e "${GREEN}🎉 Installation complete! You can now run start_server.sh${NC}" \ No newline at end of file diff --git a/pythonsetup/pipenv.sh b/pythonsetup/pipenv.sh new file mode 100755 index 0000000..e3c4a02 --- /dev/null +++ b/pythonsetup/pipenv.sh @@ -0,0 +1,22 @@ +#!/bin/bash +export PYTHONPATH=$PYTHONPATH:$(pwd)/src + + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +echo "Setting up KnowledgeCenter environment in: $SCRIPT_DIR" + +# Create virtual environment if it doesn't exist +if [ ! -d ".venv" ]; then + echo "📦 Creating Python virtual environment..." + uv venv + echo "✅ Virtual environment created" +else + echo "✅ Virtual environment already exists" +fi + +# Activate virtual environment +echo "🔄 Activating virtual environment..." +source .venv/bin/activate diff --git a/pythonsetup/pyproject.toml b/pythonsetup/pyproject.toml new file mode 100644 index 0000000..edc5eea --- /dev/null +++ b/pythonsetup/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "KnowledgeCenter" +version = "0.1.0" +description = "Add your description here" +requires-python = ">=3.13" +dependencies = [ + "beautifulsoup4>=4.13.4", + "flask>=2.3.3", + "markdown>=3.5.1", + "Werkzeug>=3.1.3", + "peewee>=3.17.0", + "pygments>=2.16.1", + "toml", + "flask_socketio", + "eventlet", + "fastapi>=0.104.0", + "uvicorn>=0.24.0", + "python-multipart>=0.0.6", + "requests>=2.31.0", + "herolib @ git+https://git.ourworld.tf/herocode/herolib_python.git", + "pudb", + "ipython" +] diff --git a/pythonsetup/start_server.sh b/pythonsetup/start_server.sh new file mode 100755 index 0000000..8df79ec --- /dev/null +++ b/pythonsetup/start_server.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# KnowledgeCenter Web Server Startup Script +# This script starts the Flask web server on port 9922 for PRODUCTION + +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +source pipenv.sh + +echo -e "${BLUE}🚀 KnowledgeCenter Web Server Startup (PRODUCTION)${NC}" +echo "=================================================" + +# Check if uv is installed +# Check if port 9922 is available +if lsof -Pi :9922 -sTCP:LISTEN -t >/dev/null 2>&1; then + echo -e "${YELLOW}⚠️ Port 9922 is already in use. Attempting to stop existing process...${NC}" + PID=$(lsof -ti:9922) + if [ ! -z "$PID" ]; then + kill -9 $PID 2>/dev/null || true + sleep 2 + echo -e "${GREEN}✅ Existing process stopped${NC}" + fi +fi + +# Set environment variables for production +export FLASK_APP=src/app.py +export FLASK_ENV=production +export FLASK_DEBUG=0 + +# Display startup information +echo "" +echo -e "${BLUE}📋 Server Information:${NC}" +echo " • Application: KnowledgeCenter Interest Registration" +echo " • Port: 9922" +echo " • URL: http://localhost:9922" +echo " • Environment: Production" +echo " • Debug Mode: Disabled" +echo "" + +# Function to handle cleanup on exit +cleanup() { + echo -e "\n${YELLOW}🛑 Shutting down server...${NC}" + exit 0 +} + +# Set trap for cleanup +trap cleanup SIGINT SIGTERM + +# Start the Flask development server +echo -e "${GREEN}🌟 Starting KnowledgeCenter web server...${NC}" +echo -e "${BLUE}📡 Server will be available at: http://localhost:9922${NC}" +echo -e "${YELLOW}💡 Press Ctrl+C to stop the server${NC}" +echo "" + +# Start the server with uv, specifying production config +uv run python src/app.py --cfg prod --port 9922 --host 0.0.0.0 + +# This line should not be reached unless the server exits +echo -e "${RED}❌ Server stopped unexpectedly${NC}" diff --git a/pythonsetup/start_server_debug.sh b/pythonsetup/start_server_debug.sh new file mode 100755 index 0000000..fcdc507 --- /dev/null +++ b/pythonsetup/start_server_debug.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# KnowledgeCenter Web Server Startup Script +# This script starts the Flask web server on port 9922 for PRODUCTION + +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +source pipenv.sh + +echo -e "${BLUE}🚀 KnowledgeCenter Web Server Startup (DEVELOPMENT/DEBUG)${NC}" +echo "=================================================" + +# Check if uv is installed +# Check if port 9922 is available +if lsof -Pi :9922 -sTCP:LISTEN -t >/dev/null 2>&1; then + echo -e "${YELLOW}⚠️ Port 9922 is already in use. Attempting to stop existing process...${NC}" + PID=$(lsof -ti:9922) + if [ ! -z "$PID" ]; then + kill -9 $PID 2>/dev/null || true + sleep 2 + echo -e "${GREEN}✅ Existing process stopped${NC}" + fi +fi + +# Set environment variables for production +export FLASK_APP=src/app.py +export FLASK_ENV=development +export FLASK_DEBUG=1 + +# Display startup information +echo "" +echo -e "${BLUE}📋 Server Information:${NC}" +echo " • Application: KnowledgeCenter Interest Registration" +echo " • Port: 9922" +echo " • URL: http://localhost:9922" +echo " • Environment: Development" +echo " • Debug Mode: Enabled" +echo "" + +# Function to handle cleanup on exit +cleanup() { + echo -e "\n${YELLOW}🛑 Shutting down server...${NC}" + exit 0 +} + +# Set trap for cleanup +trap cleanup SIGINT SIGTERM + +# Start the Flask development server +echo -e "${GREEN}🌟 Starting KnowledgeCenter web server...${NC}" +echo -e "${BLUE}📡 Server will be available at: http://localhost:9922${NC}" +echo -e "${YELLOW}💡 Press Ctrl+C to stop the server${NC}" +echo "" + +# Start the server with uv, specifying production config +uv run python src/app.py --cfg dev --port 9922 --host 0.0.0.0 --debug + +# This line should not be reached unless the server exits +echo -e "${RED}❌ Server stopped unexpectedly${NC}"