{
  "benchmarkId": "plc-simulator-benchmark-2026",
  "protocolVersion": "0.1.0",
  "publishedDate": "2026-07-24",
  "publisher": "PLC Programming IO",
  "status": "protocol_published_results_pending",
  "statusStatement": "No product runs, raw results, scores, rankings, screenshots, or benchmark conclusions have been published.",
  "commercialDisclosure": "PLC Programming IO promotes PLC Simulation Software and may benefit commercially from referrals. The product must use the same frozen cases, weights, evidence requirements, and retest rules as every other product.",
  "productRoster": {
    "status": "not_frozen",
    "requiredFields": [
      "product_name",
      "product_url",
      "version_or_build",
      "test_date",
      "operating_environment",
      "license_tier",
      "price_and_currency",
      "price_check_date",
      "inclusion_reason",
      "commercial_relationship"
    ]
  },
  "allowedOutcomes": [
    "pass",
    "fail",
    "unsupported",
    "blocked",
    "not_tested"
  ],
  "runRules": {
    "executionRepetitions": 3,
    "preserveFirstRun": true,
    "missingEvidenceOutcome": "not_tested",
    "retestRequiresPublishedReason": true,
    "weightsFrozenBeforeResults": true
  },
  "scoring": [
    {
      "group": "execution_correctness",
      "weightPercent": 30
    },
    {
      "group": "reproducibility_and_diagnostics",
      "weightPercent": 20
    },
    {
      "group": "dialect_and_instruction_coverage",
      "weightPercent": 20
    },
    {
      "group": "project_portability",
      "weightPercent": 10
    },
    {
      "group": "access_and_time_to_first_run",
      "weightPercent": 10
    },
    {
      "group": "documentation_and_limitations",
      "weightPercent": 10
    }
  ],
  "requiredResultFields": [
    "benchmark_id",
    "protocol_version",
    "product_name",
    "product_version",
    "license_tier",
    "environment",
    "test_case_id",
    "run_number",
    "started_at",
    "completed_at",
    "outcome",
    "observed_result",
    "evidence_urls",
    "tester",
    "deviations",
    "retest_of"
  ],
  "testCases": [
    {
      "id": "ENV-001",
      "category": "access",
      "title": "Reach a runnable project",
      "objective": "Measure the documented steps and elapsed time from the published starting state to a runnable blank project.",
      "expectedOutcome": "A project reaches run mode or the product is recorded as blocked with the exact prerequisite.",
      "evidenceRequired": [
        "start_state",
        "ordered_steps",
        "elapsed_time_seconds",
        "version_screen",
        "final_run_state"
      ]
    },
    {
      "id": "LOGIC-001",
      "category": "execution_correctness",
      "title": "Stop-priority motor seal-in",
      "objective": "Implement a start, normally closed stop, seal-in contact, and motor output using the product's native equivalent.",
      "expectedOutcome": "A momentary start latches the output; stop clears it; simultaneous start and stop leaves the output off.",
      "evidenceRequired": [
        "program_file_or_listing",
        "input_sequence",
        "observed_output_trace"
      ]
    },
    {
      "id": "TIMER-001",
      "category": "execution_correctness",
      "title": "On-delay timer boundary",
      "objective": "Test false, timing, preset-boundary, done, and reset states for an on-delay timer.",
      "expectedOutcome": "The output remains false before the preset, changes at the documented boundary, and resets according to the product's documented semantics.",
      "evidenceRequired": [
        "timer_configuration",
        "timebase",
        "preset",
        "timestamped_state_trace"
      ]
    },
    {
      "id": "COUNTER-001",
      "category": "execution_correctness",
      "title": "Counter increment and reset",
      "objective": "Apply discrete false-to-true events, verify accumulated value and done state, then reset.",
      "expectedOutcome": "One increment occurs per qualifying event; done state and reset match the documented instruction semantics.",
      "evidenceRequired": [
        "counter_configuration",
        "input_event_log",
        "accumulator_trace",
        "reset_trace"
      ]
    },
    {
      "id": "EDGE-001",
      "category": "execution_correctness",
      "title": "One-scan rising edge",
      "objective": "Hold an input true across multiple scans and observe a rising-edge output.",
      "expectedOutcome": "The edge output is true for one scan after a false-to-true transition and remains false until another qualifying transition.",
      "evidenceRequired": [
        "program_file_or_listing",
        "scan_trace",
        "input_transition_log"
      ]
    },
    {
      "id": "SCAN-001",
      "category": "reproducibility_and_diagnostics",
      "title": "Deterministic scan order",
      "objective": "Use two dependent networks or rungs to expose the product's documented execution order.",
      "expectedOutcome": "Three repeated runs produce the same result and the observed order is documented.",
      "evidenceRequired": [
        "program_file_or_listing",
        "three_run_traces",
        "documented_execution_model"
      ]
    },
    {
      "id": "FAULT-001",
      "category": "reproducibility_and_diagnostics",
      "title": "Invalid arithmetic handling",
      "objective": "Attempt a divide-by-zero or closest supported invalid arithmetic case without physical outputs.",
      "expectedOutcome": "The product blocks the operation, reports a diagnosable error, or produces a documented runtime outcome without concealing the fault.",
      "evidenceRequired": [
        "test_expression",
        "diagnostic_message",
        "runtime_state",
        "recovery_steps"
      ]
    },
    {
      "id": "PROJECT-001",
      "category": "project_portability",
      "title": "Save, close, reopen, and rerun",
      "objective": "Persist the motor seal-in project, close the product, reopen the saved project, and repeat the input sequence.",
      "expectedOutcome": "Logic, tags, instruction settings, and observed behaviour remain intact or the limitation is explicitly documented.",
      "evidenceRequired": [
        "saved_artifact",
        "before_and_after_project_state",
        "repeat_run_trace"
      ]
    },
    {
      "id": "PROJECT-002",
      "category": "project_portability",
      "title": "Export or share a project",
      "objective": "Use the product's documented export or sharing path and record format and import limitations.",
      "expectedOutcome": "The artifact can be reopened under the documented scope, or export is recorded as unsupported.",
      "evidenceRequired": [
        "export_steps",
        "artifact_metadata",
        "reopen_or_import_result",
        "documented_limitations"
      ]
    },
    {
      "id": "DOCS-001",
      "category": "documentation_and_limitations",
      "title": "Locate versioned capability documentation",
      "objective": "Find official documentation for supported instructions, dialects, tiers, and known limitations.",
      "expectedOutcome": "Claims are traceable to versioned official documentation or are marked not verified.",
      "evidenceRequired": [
        "source_urls",
        "source_versions_or_dates",
        "capability_claims",
        "unresolved_conflicts"
      ]
    }
  ]
}