A collection of notes and resources
データの完全性を確保し、モデリングに向けてデータを準備する
| DPL = P(y=1 | a) - P(y=1 | d) |
SageMaker Clarify を使用：
from sagemaker import clarify
bias_config = clarify.BiasConfig(
label_values_or_threshold=[1],
facet_name='gender',
facet_values_or_threshold=[0]
)
clarify_processor = clarify.SageMakerClarifyProcessor(
role=role,
instance_count=1,
instance_type='ml.m5.xlarge',
sagemaker_session=session
)
clarify_processor.run_pre_training_bias(
data_config=data_config,
data_bias_config=bias_config,
methods='all'
)
from sklearn.model_selection import train_test_split
# 層化分割
X_train, X_test, y_train, y_test = train_test_split(
X, y,
test_size=0.2,
stratify=y,
random_state=42
)
# さらに検証セットを分割
X_train, X_val, y_train, y_val = train_test_split(
X_train, y_train,
test_size=0.25, # 0.25 x 0.8 = 0.2 of total
stratify=y_train,
random_state=42
)
import boto3
glue_client = boto3.client('glue')
# データ品質ルールセットの作成
response = glue_client.create_data_quality_ruleset(
Name='customer-data-quality',
Ruleset='''
Rules = [
RowCount > 1000,
IsComplete "customer_id",
IsUnique "customer_id",
ColumnValues "age" between 0 and 120,
ColumnValues "email" matches "^[\\w\\.]+@[\\w\\.]+$"
]
''',
TargetTable={
'DatabaseName': 'customer_db',
'TableName': 'customers'
}
)
from sagemaker.tensorflow import TensorFlow
estimator = TensorFlow(
entry_point='train.py',
role=role,
instance_type='ml.p3.2xlarge',
instance_count=2,
framework_version='2.12',
subnets=['subnet-xxx'],
security_group_ids=['sg-xxx'],
file_system_id='fs-xxx', # EFS ID
file_system_type='EFS',
file_system_directory_path='/ml-data',
file_system_access_mode='ro'
)