# Create the diagram object
withdiagrams.Diagram("Programming Languages",show=False,filename="languages"):# Get all the languages available in this library
languages=[itemforitemindir(diagrams.programming.language)ifitem[0]!='_']# Divide the representation in two lines
mid_index=len(languages)//2first_line=languages[:mid_index]second_line=languages[mid_index:]# Add nodes in the first row
prev_node=Noneforlanguageinfirst_line:current_node=eval(f"diagrams.programming.language.{language}(language)")ifprev_nodeisnotNone:prev_node>>current_nodeprev_node=current_node# Add nodes in the second row
prev_node=Noneforlanguageinsecond_line:current_node=eval(f"diagrams.programming.language.{language}(language)")ifprev_nodeisnotNone:prev_node>>current_nodeprev_node=current_nodeImage("languages.png")
fromdiagramsimportDiagram,Clusterfromdiagrams.aws.storageimportS3fromdiagrams.aws.analyticsimportGlue,Athenaimportdiagrams.aws.mlasmlfromdiagrams.aws.integrationimportStepFunctionsfromdiagrams.aws.computeimportLambdafromdiagrams.aws.networkimportAPIGatewayfromIPython.displayimportImagewithDiagram("AWS Data Processing Pipeline",show=False):lambda_raw=Lambda('Get Raw Data')# Buckets de S3
withCluster("Data Lake"):s3_rawData=S3("raw_data")s3_stage=S3("staging_data")s3_data_capture=S3("data_capture")athena=Athena("Athena")s3_rawData>>athenas3_stage>>athenas3_data_capture>>athena# Step Functions Pipeline
withCluster("Data Processing Pipeline"):step_functions=StepFunctions("Pipeline")# Glue Jobs in Step Functions
withCluster("Glue Jobs"):data_quality=Glue("job_data_quality")transform=Glue("job_data_transform")dataset_preparation=Glue("job_dataset_model")# Define Step Functions Flows
step_functions>>data_quality>>transform>>dataset_preparations3_rawData>>data_quality# SageMaker for model training and deployment
withCluster("SageMaker Model Deployment"):train_model=ml.SagemakerTrainingJob("job_train_model")eval_model=ml.SagemakerGroundTruth("job_evaluate_model")endpoint=ml.SagemakerModel("model_enpoint")# API Gateway and Lambda for the endpoint
api_gateway=APIGateway("API_gateway")lambda_fn=Lambda("invoke_endpoint")# Connection
lambda_raw>>s3_rawDatas3_stage>>train_model>>eval_model>>endpointendpoint>>lambda_fn>>api_gatewayendpoint>>s3_data_capturedataset_preparation>>train_modelImage("aws_data_processing_pipeline.png")