add timeseries endpoints and update server.ts

/api/series/auto-arima-find: find parameters of SARIMA model automatically
/api/series/manual-forecast: use determined model with parameters to forecast next values
/api/series/identify-correlations: Calculate ACF and PACF for a time series
/api/series/decompose-stl: Applies Seasonal-Trend-Loess (STL) decomposition to separate the series into trend, seasonal, and residual components.
This commit is contained in:
raymond 2025-09-12 02:46:52 +00:00
parent 13f3c7b053
commit b1b2dcf18c
3 changed files with 682 additions and 26 deletions

133
analysis_pipelines.ts Normal file
View file

@ -0,0 +1,133 @@
// analysis_pipelines.ts - High-level workflows for common analysis tasks.
import { SignalProcessor } from './signal_processing_convolution';
import { TimeSeriesAnalyzer, STLDecomposition } from './timeseries';
/**
* The comprehensive result of a denoise and detrend operation.
*/
export interface DenoiseAndDetrendResult {
original: number[];
smoothed: number[];
decomposition: STLDecomposition;
}
/**
* The result of an automatic SARIMA parameter search.
*/
export interface AutoArimaResult {
bestModel: {
p: number;
d: number;
q: number;
P: number;
D: number;
Q: number;
s: number; // Correctly included
aic: number;
};
searchLog: { p: number; d: number; q: number; P: number; D: number; Q: number; s: number; aic: number }[];
}
/**
* A class containing high-level analysis pipelines that combine
* functions from various processing libraries.
*/
export class AnalysisPipelines {
/**
* A full pipeline to take a raw signal, smooth it to remove noise,
* and then decompose it into trend, seasonal, and residual components.
* @param series The original time series data.
* @param period The seasonal period for STL decomposition.
* @param smoothWindow The window size for the initial smoothing (denoising) pass.
* @returns An object containing the original, smoothed, and decomposed series.
*/
static denoiseAndDetrend(series: number[], period: number, smoothWindow: number = 5): DenoiseAndDetrendResult {
// Ensure window is odd for symmetry
if (smoothWindow > 1 && smoothWindow % 2 === 0) {
smoothWindow++;
}
const smoothed = SignalProcessor.smooth(series, {
method: 'gaussian',
windowSize: smoothWindow
});
const decomposition = TimeSeriesAnalyzer.stlDecomposition(smoothed, period);
return {
original: series,
smoothed: smoothed,
decomposition: decomposition,
};
}
/**
* [FINAL CORRECTED VERSION] Performs a full grid search to find the optimal SARIMA parameters.
* This version now correctly includes 's' in the final result object.
* @param series The original time series data.
* @param seasonalPeriod The seasonal period of the data (e.g., 7 for weekly, 12 for monthly).
* @returns An object containing the best model parameters and a log of the search.
*/
static findBestArimaParameters(
series: number[],
seasonalPeriod: number,
maxD: number = 1,
maxP: number = 2,
maxQ: number = 2,
maxSeasonalD: number = 1,
maxSeasonalP: number = 2,
maxSeasonalQ: number = 2
): AutoArimaResult {
const searchLog: any[] = [];
let bestModel: any = { aic: Infinity };
const calculateAIC = (residuals: number[], numParams: number): number => {
const n = residuals.length;
if (n === 0) return Infinity;
const sse = residuals.reduce((sum, r) => sum + r * r, 0);
if (sse < 1e-9) return -Infinity; // Perfect fit
const logLikelihood = -n / 2 * (Math.log(2 * Math.PI) + Math.log(sse / n)) - n / 2;
return 2 * numParams - 2 * logLikelihood;
};
// Grid search over all parameter combinations
for (let d = 0; d <= maxD; d++) {
for (let p = 0; p <= maxP; p++) {
for (let q = 0; q <= maxQ; q++) {
for (let D = 0; D <= maxSeasonalD; D++) {
for (let P = 0; P <= maxSeasonalP; P++) {
for (let Q = 0; Q <= maxSeasonalQ; Q++) {
// Skip trivial models where nothing is done
if (p === 0 && d === 0 && q === 0 && P === 0 && D === 0 && Q === 0) continue;
const options = { p, d, q, P, D, Q, s: seasonalPeriod };
try {
const { residuals } = TimeSeriesAnalyzer.arimaForecast(series, options, 0);
const numParams = p + q + P + Q;
const aic = calculateAIC(residuals, numParams);
// Construct the full model info object, ensuring 's' is included
const modelInfo = { p, d, q, P, D, Q, s: seasonalPeriod, aic };
searchLog.push(modelInfo);
if (modelInfo.aic < bestModel.aic) {
bestModel = modelInfo;
}
} catch (error) {
// Skip invalid parameter combinations that cause errors
}
} } } } } }
if (bestModel.aic === Infinity) {
throw new Error("Could not find a suitable SARIMA model. The data may be too short or complex.");
}
// Sort the log by AIC for easier reading
searchLog.sort((a, b) => a.aic - b.aic);
return { bestModel, searchLog };
}
}

229
server.ts
View file

@ -8,19 +8,20 @@ import swaggerJsdoc from 'swagger-jsdoc';
import swaggerUi from 'swagger-ui-express';
import * as math from 'mathjs';
import * as _ from 'lodash';
import cors from 'cors'; // <-- 1. IMPORT THE CORS PACKAGE
// These imports assume the files exist in the same directory
// Assuming these files exist in the same directory
// import { KMeans, KMeansOptions } from './kmeans';
// import { getWeekNumber, getSameWeekDayLastYear } from './time-helper';
// import { calculateLinearRegression, generateForecast, calculatePredictionIntervals, ForecastResult } from './prediction';
import { SignalProcessor, SmoothingOptions, EdgeDetectionOptions } from './signal_processing_convolution';
import { convolve1D, ConvolutionKernels } from './convolution'; // Direct import for new functions
import { TimeSeriesAnalyzer, ARIMAOptions } from './timeseries';
import { AnalysisPipelines } from './analysis_pipelines';
import { convolve1D, convolve2D, ConvolutionKernels } from './convolution';
// Dummy interfaces/classes if the files are not present, to prevent compile errors
interface KMeansOptions {}
class KMeans {
constructor(p: any, n: any, o: any) {}
run = () => ({ clusters: [] })
}
class KMeans { constructor(p: any, n: any, o: any) {}; run = () => ({ clusters: [] }) }
const getWeekNumber = (d: string) => 1;
const getSameWeekDayLastYear = (d: string) => new Date().toISOString();
interface ForecastResult {}
@ -28,25 +29,27 @@ const calculateLinearRegression = (v: any) => ({slope: 1, intercept: 0});
const generateForecast = (m: any, l: any, p: any) => [];
const calculatePredictionIntervals = (v: any, m: any, f: any) => [];
const app = express();
app.use(express.json());
app.use(cors()); // <-- 2. ENABLE CORS FOR ALL ROUTES
const PORT = process.env.PORT || 3000;
const swaggerOptions = {
swaggerDefinition: {
openapi: '3.0.0',
info: {
title: 'My Express API',
version: '1.0.0',
description: 'API documentation for my awesome Express app',
},
servers: [
{
url: `http://localhost:${PORT}`,
},
],
swaggerDefinition: {
openapi: '3.0.0',
info: {
title: 'My Express API',
version: '1.0.0',
description: 'API documentation for my awesome Express app',
},
apis: ["./server.ts"], // Pointing to this file for Swagger docs
servers: [
{
url: `http://localhost:${PORT}`,
},
],
},
apis: ["./server_convolution.ts"], // Pointing to the correct, renamed file
};
const swaggerSpec = swaggerJsdoc(swaggerOptions);
@ -67,7 +70,6 @@ interface DataMatrix {
columns?: string[];
rows?: string[];
}
interface Condition {
field: string;
operator: '>' | '<' | '=' | '>=' | '<=' | '!=';
@ -85,9 +87,8 @@ interface ApiResponse<T> {
// ========================================
const handleError = (error: unknown): string => {
return error instanceof Error ? error.message : 'Unknown error';
return error instanceof Error ? error.message : 'Unknown error';
};
const validateSeries = (series: DataSeries): void => {
if (!series || !Array.isArray(series.values) || series.values.length === 0) {
throw new Error('Series must contain at least one value');
@ -854,6 +855,159 @@ app.post('/api/series/rolling', (req, res) => {
}
});
/**
* @swagger
* /api/series/auto-arima-find:
* post:
* summary: (EXPERIMENTAL) Automatically find best SARIMA parameters
* description: Performs a grid search to find the best SARIMA parameters based on AIC. NOTE - This is a simplified estimation and may not find the true optimal model. For best results, use the identification tools and the 'manual-forecast' endpoint.
* tags: [Series Operations]
* requestBody:
* required: true
* content:
* application/json:
* schema:
* type: object
* properties:
* series:
* $ref: '#/components/schemas/DataSeries'
* seasonalPeriod:
* type: integer
* description: The seasonal period of the data (e.g., 7 for weekly).
* example: 7
* responses:
* '200':
* description: The best model found and the search log.
* '400':
* description: Invalid input data.
*/
app.post('/api/series/auto-arima-find', (req, res) => {
try {
const { series, seasonalPeriod } = req.body;
validateSeries(series);
const result = AnalysisPipelines.findBestArimaParameters(series.values, seasonalPeriod);
res.status(200).json({ success: true, data: result });
} catch (error) {
const errorMessage = handleError(error);
res.status(400).json({ success: false, error: errorMessage });
}
});
/**
* @swagger
* /api/series/manual-forecast:
* post:
* summary: Generate a forecast with manually specified SARIMA parameters
* description: This is the primary forecasting tool. It allows an expert user (who has analyzed ACF/PACF plots) to apply a specific SARIMA model to a time series and generate a forecast.
* tags: [Series Operations]
* requestBody:
* required: true
* content:
* application/json:
* schema:
* type: object
* properties:
* series:
* $ref: '#/components/schemas/DataSeries'
* options:
* $ref: '#/components/schemas/ARIMAOptions'
* forecastSteps:
* type: integer
* description: The number of future time steps to predict.
* example: 7
* responses:
* '200':
* description: The forecast results.
* '400':
* description: Invalid input data
*/
app.post('/api/series/manual-forecast', (req, res) => {
try {
const { series, options, forecastSteps } = req.body;
validateSeries(series);
const result = TimeSeriesAnalyzer.arimaForecast(series.values, options, forecastSteps);
res.status(200).json({ success: true, data: result });
} catch (error) {
const errorMessage = handleError(error);
res.status(400).json({ success: false, error: errorMessage });
}
});
/**
* @swagger
* /api/series/identify-correlations:
* post:
* summary: Calculate ACF and PACF for a time series
* description: Returns the Autocorrelation and Partial Autocorrelation function values, which are essential for identifying SARIMA model parameters.
* tags: [Series Operations]
* requestBody:
* required: true
* content:
* application/json:
* schema:
* type: object
* properties:
* series:
* $ref: '#/components/schemas/DataSeries'
* maxLag:
* type: integer
* description: The maximum number of lags to calculate.
* example: 40
* responses:
* '200':
* description: The calculated ACF and PACF values.
* '400':
* description: Invalid input data.
*/
app.post('/api/series/identify-correlations', (req, res) => {
try {
const { series, maxLag } = req.body;
validateSeries(series);
const acf = TimeSeriesAnalyzer.calculateACF(series.values, maxLag);
const pacf = TimeSeriesAnalyzer.calculatePACF(series.values, maxLag);
res.status(200).json({ success: true, data: { acf, pacf } });
} catch (error) {
res.status(400).json({ success: false, error: handleError(error) });
}
});
/**
* @swagger
* /api/series/decompose-stl:
* post:
* summary: Decompose a time series into components
* description: Applies Seasonal-Trend-Loess (STL) decomposition to separate the series into trend, seasonal, and residual components.
* tags: [Series Operations]
* requestBody:
* required: true
* content:
* application/json:
* schema:
* type: object
* properties:
* series:
* $ref: '#/components/schemas/DataSeries'
* period:
* type: integer
* description: The seasonal period of the data (e.g., 7 for weekly).
* example: 7
* responses:
* '200':
* description: The decomposed components of the time series.
* '400':
* description: Invalid input data.
*/
app.post('/api/series/decompose-stl', (req, res) => {
try {
const { series, period } = req.body;
validateSeries(series);
const result = TimeSeriesAnalyzer.stlDecomposition(series.values, period);
res.status(200).json({ success: true, data: result });
} catch (error) {
res.status(400).json({ success: false, error: handleError(error) });
}
});
/**
* @swagger
* /api/ml/kmeans:
@ -1648,6 +1802,30 @@ app.get('/api/kernels/:name', (req, res) => {
* type: number
* default: 0.1
* description: The sensitivity threshold for detecting an edge. Values below this will be set to 0.
* ARIMAOptions:
* type: object
* properties:
* p:
* type: integer
* description: Non-seasonal AutoRegressive (AR) order.
* d:
* type: integer
* description: Non-seasonal Differencing (I) order.
* q:
* type: integer
* description: Non-seasonal Moving Average (MA) order.
* P:
* type: integer
* description: Seasonal AR order.
* D:
* type: integer
* description: Seasonal Differencing order.
* Q:
* type: integer
* description: Seasonal MA order.
* s:
* type: integer
* description: The seasonal period length (e.g., 7 for weekly).
* ApiResponse:
* type: object
* properties:
@ -1780,7 +1958,7 @@ app.get('/api/docs/export/html', (req, res) => {
app.use((err: Error, req: express.Request, res: express.Response, next: express.NextFunction) => {
console.error(err.stack);
res.status(500).json({ success: false, error: 'Internal server error' } as ApiResponse<any>);
res.status(500).json({ success: false, error: 'Internal server error' });
});
app.use('*', (req, res) => {
@ -1792,9 +1970,8 @@ app.use('*', (req, res) => {
// ========================================
app.listen(PORT, () => {
console.log(`Analytics API server running on port ${PORT}`);
console.log(`Health check: http://localhost:${PORT}/api/health`);
console.log(`API Documentation: http://localhost:${PORT}/api-docs`);
console.log(`Analytics API server running on port ${PORT}`);
console.log(`API Documentation: http://localhost:${PORT}/api-docs`);
});
export default app;

346
timeseries.ts Normal file
View file

@ -0,0 +1,346 @@
// timeseries.ts - A library for time series analysis, focusing on ARIMA.
// ========================================
// TYPE DEFINITIONS
// ========================================
/**
* Defines the parameters for an ARIMA model.
* (p, d, q) are the non-seasonal components.
* (P, D, Q, s) are the optional seasonal components for SARIMA.
*/
export interface ARIMAOptions {
p: number; // AutoRegressive (AR) order
d: number; // Differencing (I) order
q: number; // Moving Average (MA) order
P?: number; // Seasonal AR order
D?: number; // Seasonal Differencing order
Q?: number; // Seasonal MA order
s?: number; // Seasonal period length
}
/**
* The result object from an ARIMA forecast.
*/
export interface ARIMAForecastResult {
forecast: number[]; // The predicted future values
residuals: number[]; // The errors of the model fit on the original data
model: ARIMAOptions; // The model parameters used
}
/**
* The result object from an STL decomposition.
*/
export interface STLDecomposition {
seasonal: number[]; // The seasonal component of the series
trend: number[]; // The trend component of the series
residual: number[]; // The remainder/residual component
original: number[]; // The original series, for comparison
}
/**
* A class for performing time series analysis, including identification and forecasting.
*/
export class TimeSeriesAnalyzer {
// ========================================
// 1. IDENTIFICATION METHODS
// ========================================
/**
* Calculates the difference of a time series.
* This is the 'I' (Integrated) part of ARIMA, used to make a series stationary.
* @param series The input data series.
* @param lag The lag to difference by (usually 1).
* @returns A new, differenced time series.
*/
static difference(series: number[], lag: number = 1): number[] {
if (lag < 1 || !Number.isInteger(lag)) {
throw new Error('Lag must be a positive integer.');
}
if (series.length <= lag) {
return [];
}
const differenced: number[] = [];
for (let i = lag; i < series.length; i++) {
differenced.push(series[i] - series[i - lag]);
}
return differenced;
}
/**
* Helper function to calculate the autocovariance of a series at a given lag.
*/
private static autocovariance(series: number[], lag: number): number {
const n = series.length;
if (lag >= n) return 0;
const mean = series.reduce((a, b) => a + b) / n;
let sum = 0;
for (let i = lag; i < n; i++) {
sum += (series[i] - mean) * (series[i - lag] - mean);
}
return sum / n;
}
/**
* Calculates the Autocorrelation Function (ACF) for a time series.
* ACF helps in determining the 'q' parameter for an ARIMA model.
* @param series The input data series.
* @param maxLag The maximum number of lags to calculate.
* @returns An array of correlation values from lag 1 to maxLag.
*/
static calculateACF(series: number[], maxLag: number): number[] {
if (series.length < 2) return [];
const variance = this.autocovariance(series, 0);
if (variance === 0) {
return new Array(maxLag).fill(1);
}
const acf: number[] = [];
for (let lag = 1; lag <= maxLag; lag++) {
acf.push(this.autocovariance(series, lag) / variance);
}
return acf;
}
/**
* Calculates the Partial Autocorrelation Function (PACF) for a time series.
* This now uses the Durbin-Levinson algorithm for an accurate calculation.
* PACF helps in determining the 'p' parameter for an ARIMA model.
* @param series The input data series.
* @param maxLag The maximum number of lags to calculate.
* @returns An array of partial correlation values from lag 1 to maxLag.
*/
static calculatePACF(series: number[], maxLag: number): number[] {
const acf = this.calculateACF(series, maxLag);
const pacf: number[] = [];
if (acf.length === 0) return [];
pacf.push(acf[0]); // PACF at lag 1 is the same as ACF at lag 1
for (let k = 2; k <= maxLag; k++) {
let numerator = acf[k - 1];
let denominator = 1;
const phi = new Array(k + 1).fill(0).map(() => new Array(k + 1).fill(0));
for(let i=1; i<=k; i++) {
phi[i][i] = acf[i-1];
}
for (let j = 1; j < k; j++) {
const factor = pacf[j - 1];
numerator -= factor * acf[k - j - 1];
denominator -= factor * acf[j - 1];
}
if (Math.abs(denominator) < 1e-9) { // Avoid division by zero
pacf.push(0);
continue;
}
const pacf_k = numerator / denominator;
pacf.push(pacf_k);
}
return pacf;
}
/**
* Decomposes a time series using the robust Classical Additive method.
* This version correctly isolates trend, seasonal, and residual components.
* @param series The input data series.
* @param period The seasonal period (e.g., 7 for daily data with a weekly cycle).
* @returns An object containing the seasonal, trend, and residual series.
*/
static stlDecomposition(series: number[], period: number): STLDecomposition {
if (series.length < 2 * period) {
throw new Error("Series must be at least twice the length of the seasonal period.");
}
// Helper for a centered moving average
const movingAverage = (data: number[], window: number) => {
const result = [];
const halfWindow = Math.floor(window / 2);
for (let i = 0; i < data.length; i++) {
const start = Math.max(0, i - halfWindow);
const end = Math.min(data.length, i + halfWindow + 1);
let sum = 0;
for (let j = start; j < end; j++) {
sum += data[j];
}
result.push(sum / (end - start));
}
return result;
};
// Step 1: Calculate the trend using a centered moving average.
// If period is even, we use a 2x-MA to center it correctly.
let trend: number[];
if (period % 2 === 0) {
const intermediate = movingAverage(series, period);
trend = movingAverage(intermediate, 2);
} else {
trend = movingAverage(series, period);
}
// Step 2: Detrend the series
const detrended = series.map((val, i) => val - trend[i]);
// Step 3: Calculate the seasonal component by averaging the detrended values for each period
const seasonalAverages = new Array(period).fill(0);
const seasonalCounts = new Array(period).fill(0);
for (let i = 0; i < series.length; i++) {
if (!isNaN(detrended[i])) {
const seasonIndex = i % period;
seasonalAverages[seasonIndex] += detrended[i];
seasonalCounts[seasonIndex]++;
}
}
for (let i = 0; i < period; i++) {
seasonalAverages[i] /= seasonalCounts[i];
}
// Center the seasonal component to have a mean of zero
const seasonalMean = seasonalAverages.reduce((a, b) => a + b, 0) / period;
const centeredSeasonalAverages = seasonalAverages.map(avg => avg - seasonalMean);
const seasonal = new Array(series.length).fill(0);
for (let i = 0; i < series.length; i++) {
seasonal[i] = centeredSeasonalAverages[i % period];
}
// Step 4: Calculate the residual component
const residual = detrended.map((val, i) => val - seasonal[i]);
return {
original: series,
seasonal,
trend,
residual,
};
}
// ========================================
// 2. FORECASTING METHODS
// ========================================
/**
* [UPGRADED] Generates a forecast using a simplified SARIMA model.
* This implementation now handles both non-seasonal (p,d,q) and seasonal (P,D,Q,s) components.
* @param series The input time series data.
* @param options The SARIMA parameters.
* @param forecastSteps The number of future steps to predict.
* @returns An object containing the forecast and model residuals.
*/
static arimaForecast(series: number[], options: ARIMAOptions, forecastSteps: number): ARIMAForecastResult {
const { p, d, q, P = 0, D = 0, Q = 0, s = 0 } = options;
if (series.length < p + d + (P + D) * s + q + Q * s) {
throw new Error("Data series is too short for the specified SARIMA order.");
}
const originalSeries = [...series];
let differencedSeries = [...series];
const diffLog: { lag: number, values: number[] }[] = [];
// Step 1: Apply seasonal differencing 'D' times
for (let i = 0; i < D; i++) {
diffLog.push({ lag: s, values: differencedSeries.slice(-s) });
differencedSeries = this.difference(differencedSeries, s);
}
// Step 2: Apply non-seasonal differencing 'd' times
for (let i = 0; i < d; i++) {
diffLog.push({ lag: 1, values: differencedSeries.slice(-1) });
differencedSeries = this.difference(differencedSeries, 1);
}
const n = differencedSeries.length;
// Simplified coefficients
const arCoeffs = p > 0 ? new Array(p).fill(1 / p) : [];
const maCoeffs = q > 0 ? new Array(q).fill(1 / q) : [];
const sarCoeffs = P > 0 ? new Array(P).fill(1 / P) : [];
const smaCoeffs = Q > 0 ? new Array(Q).fill(1 / Q) : [];
const residuals: number[] = new Array(n).fill(0);
const fitted: number[] = new Array(n).fill(0);
// Step 3: Fit the model
const startIdx = Math.max(p, q, P * s, Q * s);
for (let t = startIdx; t < n; t++) {
// Non-seasonal AR
let arVal = 0;
for (let i = 0; i < p; i++) arVal += arCoeffs[i] * differencedSeries[t - 1 - i];
// Non-seasonal MA
let maVal = 0;
for (let i = 0; i < q; i++) maVal += maCoeffs[i] * residuals[t - 1 - i];
// Seasonal AR
let sarVal = 0;
for (let i = 0; i < P; i++) sarVal += sarCoeffs[i] * differencedSeries[t - s * (i + 1)];
// Seasonal MA
let smaVal = 0;
for (let i = 0; i < Q; i++) smaVal += smaCoeffs[i] * residuals[t - s * (i + 1)];
fitted[t] = arVal + maVal + sarVal + smaVal;
residuals[t] = differencedSeries[t] - fitted[t];
}
// Step 4: Generate the forecast
const forecastDifferenced: number[] = [];
const extendedSeries = [...differencedSeries];
const extendedResiduals = [...residuals];
for (let f = 0; f < forecastSteps; f++) {
const t = n + f;
let nextForecast = 0;
// AR
for (let i = 0; i < p; i++) nextForecast += arCoeffs[i] * extendedSeries[t - 1 - i];
// MA (future residuals are 0)
for (let i = 0; i < q; i++) nextForecast += maCoeffs[i] * extendedResiduals[t - 1 - i];
// SAR
for (let i = 0; i < P; i++) nextForecast += sarCoeffs[i] * extendedSeries[t - s * (i + 1)];
// SMA
for (let i = 0; i < Q; i++) nextForecast += smaCoeffs[i] * extendedResiduals[t - s * (i + 1)];
forecastDifferenced.push(nextForecast);
extendedSeries.push(nextForecast);
extendedResiduals.push(0);
}
// Step 5: Invert the differencing
let forecast = [...forecastDifferenced];
for (let i = diffLog.length - 1; i >= 0; i--) {
const { lag, values } = diffLog[i];
const inverted = [];
const fullHistory = [...originalSeries, ...forecast]; // Need a temporary full history for inversion
// A simpler inversion method for forecasting
let history = [...series];
for (const forecastVal of forecast) {
const lastSeasonalVal = history[history.length - lag];
const invertedVal = forecastVal + lastSeasonalVal;
inverted.push(invertedVal);
history.push(invertedVal);
}
forecast = inverted;
}
return {
forecast,
residuals,
model: options,
};
}
}