バリデーション機能

import json
from pydantic import ValidationError

from rdetoolkit.validation import InvoiceValidator
from rdetoolkit.exceptions import InvoiceSchemaValidationError

# スキーマ定義
schema = {
    "$schema": "https://json-schema.org/draft/2020-12/schema",
    "$id": "https://rde.nims.go.jp/rde/dataset-templates/dataset_template_custom_sample/invoice.schema.json",
    "description": "RDEデータセットテンプレートサンプル固有情報invoice",
    "type": "object",
    "required": ["custom", "sample"],
    "properties": {
        "custom": {
            "type": "object",
            "label": {"ja": "固有情報", "en": "Custom Information"},
            "required": ["sample1"],
            "properties": {
                "sample1": {
                    "label": {"ja": "サンプル１", "en": "sample1"},
                    "type": "string",
                    "format": "date",
                    "options": {"unit": "A"}
                },
                "sample2": {
                    "label": {"ja": "サンプル２", "en": "sample2"},
                    "type": "number",
                    "options": {"unit": "b"}
                },
            },
        },
        "sample": {
            "type": "object",
            "label": {"ja": "試料情報", "en": "Sample Information"},
            "properties": {
                "generalAttributes": {
                    "type": "array",
                    "items": [
                        {
                            "type": "object",
                            "required": ["termId"],
                            "properties": {
                                "termId": {
                                    "const": "3adf9874-7bcb-e5f8-99cb-3d6fd9d7b55e"
                                }
                            }
                        }
                    ],
                },
                "specificAttributes": {"type": "array", "items": []},
            },
        },
    },
}

# データ例
data = {
    "datasetId": "1s1199df4-0d1v-41b0-1dea-23bf4dh09g12",
    "basic": {
        "dateSubmitted": "",
        "dataOwnerId": "0c233ef274f28e611de4074638b4dc43e737ab993132343532343430",
        "dataName": "test-dataset",
        "instrumentId": None,
        "experimentId": None,
        "description": None,
    },
    "custom": {"sample1": "2023-01-01", "sample2": 1.0},
    "sample": {
        "sampleId": "",
        "names": ["test"],
        "composition": None,
        "referenceUrl": None,
        "description": None,
        "generalAttributes": [
            {"termId": "3adf9874-7bcb-e5f8-99cb-3d6fd9d7b55e", "value": None}
        ],
        "specificAttributes": [],
        "ownerId": "de17c7b3f0ff5126831c2d519f481055ba466ddb6238666132316439",
    },
}

# スキーマファイルを保存
with open("temp/invoice.schema.json", "w") as f:
    json.dump(schema, f, ensure_ascii=False, indent=2)

# バリデーション実行
validator = InvoiceValidator("temp/invoice.schema.json")
try:
    validator.validate(obj=data)
    print("バリデーション成功")
except ValidationError as validation_error:
    raise InvoiceSchemaValidationError from validation_error

1. Field: required.0
   Type: literal_error
   Context: Input should be 'custom' or 'sample'

{
    "required": ["custom"], // sampleが定義されているのに含まれていない
    "properties": {
        "custom": { /* ... */ },
        "sample": { /* ... */ }
    }
}

{
    "required": ["custom", "sample"], // 両方を含める
    "properties": {
        "custom": { /* ... */ },
        "sample": { /* ... */ }
    }
}

# 上記のschemaとdataを使用
validator = InvoiceValidator("temp/invoice.schema.json")
try:
    validator.validate(obj=data)
    print("invoice.json バリデーション成功")
except ValidationError as validation_error:
    print(f"バリデーションエラー: {validation_error}")

"sample": {
    "sampleId": "de1132316439",
    "names": ["test"],
    "composition": null,
    "referenceUrl": null,
    "description": null,
    "generalAttributes": [
        {"termId": "3adf9874-7bcb-e5f8-99cb-3d6fd9d7b55e", "value": null}
    ],
    "specificAttributes": [],
    "ownerId": "de17c7b3f0ff5126831c2d519f481055ba466ddb6238666132316439"
}

"sample": {
    "sampleId": "de1132316439",
    "names": [],
    "composition": null,
    "referenceUrl": null,
    "description": null,
    "generalAttributes": [
        {"termId": "3adf9874-7bcb-e5f8-99cb-3d6fd9d7b55e", "value": null}
    ],
    "specificAttributes": [],
    "ownerId": "de17c7b3f0ff5126831c2d519f481055ba466ddb6238666132316439"
}

Error: Error in validating system standard field.
Please correct the following fields in invoice.json
Field: sample
Type: anyOf
Context: {'sampleId': '', 'names': 'test', 'generalAttributes': [...], 'specificAttributes': [], 'ownerId': ''} is not valid under any of the given schemas

Error: Error in validating system standard item in invoice.schema.json.
Please correct the following fields in invoice.json
Field: basic.dataOwnerId
Type: pattern
Context: String does not match expected pattern

from rdetoolkit.validation import MetadataValidator

# メタデータ定義ファイルのバリデーション
metadata_validator = MetadataValidator("path/to/metadata-def.json")
try:
    metadata_validator.validate_schema()
    print("metadata-def.json バリデーション成功")
except ValidationError as e:
    print(f"メタデータ定義バリデーションエラー: {e}")

# メタデータファイルのバリデーション
try:
    metadata_validator.validate_data("path/to/metadata.json")
    print("metadata.json バリデーション成功")
except ValidationError as e:
    print(f"メタデータバリデーションエラー: {e}")

from rdetoolkit import workflows

def my_dataset_function(rde):
    # データ処理ロジック
    rde.set_metadata({"status": "processed"})
    return 0

# ワークフロー実行時に自動バリデーションが実行される
try:
    result = workflows.run(my_dataset_function)
    print("ワークフロー実行成功")
except Exception as e:
    print(f"ワークフロー実行エラー（バリデーション含む）: {e}")

#!/bin/bash
# CI/CDバリデーションスクリプトの例

rdetoolkit validate all ./rde-project

EXIT_CODE=$?

if [ $EXIT_CODE -eq 0 ]; then
    echo "✓ バリデーション成功 - デプロイを続行します"
    exit 0
elif [ $EXIT_CODE -eq 1 ]; then
    echo "✗ バリデーション失敗 - データ/スキーマのエラーを確認してください"
    exit 1
elif [ $EXIT_CODE -eq 2 ]; then
    echo "✗ 設定エラー - コマンド引数とファイルパスを確認してください"
    exit 2
else
    echo "✗ 予期しない終了コード: $EXIT_CODE"
    exit $EXIT_CODE
fi

$ rdetoolkit validate invoice-schema tasksupport/invoice.schema.json
✓ VALID: tasksupport/invoice.schema.json

$ echo $?
0

$ rdetoolkit validate invoice raw/invoice.json --schema tasksupport/invoice.schema.json
✗ INVALID: raw/invoice.json

Errors:
  1. Field: basic.dataOwnerId
     Type: pattern
     Message: String does not match pattern ^[a-zA-Z0-9]{56}$

$ echo $?
1

$ rdetoolkit validate invoice-schema /nonexistent/schema.json
Error: Schema file not found: /nonexistent/schema.json

$ echo $?
2

$ rdetoolkit validate all ./rde-project
✓ Validating invoice schema...
✓ Validating metadata definition...
✓ Validating invoice data...
✓ Validating metadata...
✓ All validations passed

$ echo $?
0

name: RDE Validation

on: [push, pull_request]

jobs:
  validate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Pythonのセットアップ
        uses: actions/setup-python@v4
        with:
          python-version: '3.12'

      - name: rdetoolkitのインストール
        run: pip install rdetoolkit

      - name: RDEプロジェクトの検証
        run: |
          rdetoolkit validate all ./rde-project
          if [ $? -eq 1 ]; then
            echo "::error::RDEバリデーション失敗 - データとスキーマを確認してください"
            exit 1
          elif [ $? -eq 2 ]; then
            echo "::error::RDEバリデーション設定エラー"
            exit 2
          fi

validate:
  stage: test
  image: python:3.12
  script:
    - pip install rdetoolkit
    - rdetoolkit validate all ./rde-project
  allow_failure: false
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'

#!/bin/bash
set -e

validate_rde_project() {
    local project_dir="$1"

    echo "RDEプロジェクトを検証中: $project_dir"
    rdetoolkit validate all "$project_dir"

    local exit_code=$?
    case $exit_code in
        0)
            echo "✓ バリデーション成功"
            return 0
            ;;
        1)
            echo "✗ バリデーション失敗 - データまたはスキーマのエラー"
            return 1
            ;;
        2)
            echo "✗ 設定エラー - 引数とファイルを確認してください"
            return 2
            ;;
        *)
            echo "✗ 予期しない終了コード: $exit_code"
            return $exit_code
            ;;
    esac
}

validate_rde_project "./rde-project"

import json
from pathlib import Path
from rdetoolkit.validation import InvoiceValidator, MetadataValidator
from rdetoolkit.exceptions import InvoiceSchemaValidationError

def validate_all_files(project_dir: Path):
    """プロジェクト内の全ファイルをバリデーション"""

    # 1. invoice.schema.json のバリデーション
    schema_path = project_dir / "tasksupport" / "invoice.schema.json"
    invoice_path = project_dir / "invoice" / "invoice.json"

    try:
        invoice_validator = InvoiceValidator(schema_path)
        print("✓ invoice.schema.json バリデーション成功")

        # 2. invoice.json のバリデーション
        with open(invoice_path) as f:
            invoice_data = json.load(f)

        invoice_validator.validate(obj=invoice_data)
        print("✓ invoice.json バリデーション成功")

    except ValidationError as e:
        print(f"✗ Invoice バリデーションエラー: {e}")
        return False

    # 3. metadata-def.json のバリデーション
    metadata_def_path = project_dir / "tasksupport" / "metadata-def.json"
    metadata_path = project_dir / "metadata.json"

    try:
        metadata_validator = MetadataValidator(metadata_def_path)
        metadata_validator.validate_schema()
        print("✓ metadata-def.json バリデーション成功")

        # 4. metadata.json のバリデーション
        if metadata_path.exists():
            metadata_validator.validate_data(metadata_path)
            print("✓ metadata.json バリデーション成功")

    except ValidationError as e:
        print(f"✗ Metadata バリデーションエラー: {e}")
        return False

    print("🎉 全ファイルのバリデーション完了")
    return True

# 使用例
project_directory = Path("./my_rde_project")
validate_all_files(project_directory)