Ingest Setup

# Install the Tailstream agent with one command
curl -fsSL https://install.tailstream.io | bash

# The agent will automatically:
# - Discover your log files
# - Configure streaming to your endpoints
# - Handle authentication and retries
# - Monitor logs in real-time

# Open source: https://github.com/tailstream-io/tailstream-agent

curl -X POST https://tailstream.io/api/ingest/{streamId} \
  -H "Content-Type: application/x-ndjson" \
  -H "Authorization: Bearer <INGEST_TOKEN>" \
  --data-raw '{"ts": $(($(date +%s) * 1000)), "host": "api.example.com", "path": "/api/users/profile", "method": "GET", "status": 200, "rt": 0.123, "bytes": 1234, "src": "web-1"}'

# 1. NGINX CONFIGURATION
# Add this to your nginx.conf
log_format tailstream_json escape=json
  '{'
    '"ts":"$msec",'
    '"host":"$host",'
    '"path":"$request_uri",'
    '"status":$status,'
    '"rt":$request_time,'
    '"bytes":$bytes_sent,'
    '"method":"$request_method",'
    '"src":"$hostname",'
    '"ip":"$remote_addr",'
    '"user_agent":"$http_user_agent"'
  '}';

# Use the log format in your server block
server {
    access_log /var/log/nginx/tailstream.log tailstream_json;
    # ... rest of your server config
}

# 2. VECTOR CONFIGURATION
# Save as vector.yaml
data_dir: "/var/lib/vector"  # Default location

sources:
  nginx_logs:
    type: "file"
    include:
      - "/var/log/nginx/tailstream.log"
    read_from: "beginning"

transforms:
  parse_json_logs:
    type: "remap"
    inputs: ["nginx_logs"]
    source: |
      # Parse the JSON log from the message field
      ., err = parse_json(.message)
      if err != null {
        log("Failed to parse JSON log: " + string!(err), level: "warn")
        abort
      }

      # Ensure field types are correct for downstream processing
      .ts = if is_string(.ts) { to_float!(.ts) } else { .ts }
      .host = string!(.host)
      .path = string!(.path)
      .method = string!(.method)
      .status = to_int!(.status)
      .rt = to_float!(.rt)
      .bytes = to_int!(.bytes)
      .src = string!(.src)

sinks:
  tailstream:
    type: "http"
    inputs: ["parse_json_logs"]
    uri: "https://tailstream.io/api/ingest/{streamId}"
    method: "post"
    framing:
      method: "newline_delimited"
    encoding:
      codec: "json"
    compression: "gzip"
    healthcheck: true
    request:
      headers:
        Authorization: "Bearer <INGEST_TOKEN>"
    batch:
      max_events: 500
      timeout_secs: 1

<?php

// config/logging.php
return [
    'channels' => [
        'tailstream' => [
            'driver' => 'custom',
            'via' => App\Logging\TailstreamLogger::class,
            'endpoint' => 'https://tailstream.io/api/ingest/{streamId}',
            'token' => env('TAILSTREAM_TOKEN'),
        ],
    ],
];

---

<?php

// app/Logging/TailstreamLogger.php
namespace App\Logging;

use Monolog\Logger;
use Monolog\Handler\AbstractProcessingHandler;
use Monolog\LogRecord;

class TailstreamLogger
{
    public function __invoke(array $config): Logger
    {
        $logger = new Logger('tailstream');
        $logger->pushHandler(new TailstreamHandler($config));
        return $logger;
    }
}

class TailstreamHandler extends AbstractProcessingHandler
{
    private string $endpoint;
    private string $token;

    public function __construct(array $config)
    {
        $this->endpoint = $config['endpoint'];
        $this->token = $config['token'];
        parent::__construct();
    }

    protected function write(LogRecord $record): void
    {
        $data = [
            'ts' => $record->datetime->getTimestamp() * 1000,
            'host' => gethostname(),
            'path' => request()->path() ?? '/',
            'method' => request()->method() ?? 'GET',
            'status' => http_response_code() ?: 200,
            'rt' => 0.0, // Laravel logs don't have response time by default
            'bytes' => 0, // Laravel logs don't have response size by default
            'src' => gethostname(),
            'level' => $record->level->getName(),
            'message' => $record->message,
            'context' => $record->context,
        ];

        $this->sendToTailstream(json_encode($data));
    }

    private function sendToTailstream(string $data): void
    {
        $ch = curl_init($this->endpoint);
        curl_setopt_array($ch, [
            CURLOPT_POST => true,
            CURLOPT_POSTFIELDS => $data,
            CURLOPT_HTTPHEADER => [
                'Content-Type: application/x-ndjson',
                'Authorization: Bearer ' . $this->token,
            ],
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_TIMEOUT => 5,
        ]);
        curl_exec($ch);
        curl_close($ch);
    }
}