-
Notifications
You must be signed in to change notification settings - Fork 2.9k
/
AwsS3BucketAPILogsParser.txt
101 lines (101 loc) · 19.9 KB
/
AwsS3BucketAPILogsParser.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
// KQL AWS S3 Bucket API Logs Parser
// Last Updated Date: June 22, 2020
//
// Enable AWS S3 Object Level Logging:
// https://docs.aws.amazon.com/AmazonS3/latest/user-guide/enable-cloudtrail-events.html
//
// Parser Notes:
// 1. This parser works if logs are ingested via Logstash config under Logstash folder.
// 2. Replace the custom log table name with your table name
//
// Usage Instruction :
// Paste below query in log analytics, click on Save button and select as Function from drop down by specifying function name and alias (e.g. AWSS3BucketAPILogParsed).
// Function usually takes 10-15 minutes to activate. You can then use function alias from any other queries (e.g. AWSS3BucketAPILogParsed | take 10).
// Reference : Using functions in Azure monitor log queries : https://docs.microsoft.com/azure/azure-monitor/log-query/functions
//
//
AwsBucketAPILogs_CL
| where logFiles_s == "" //Filtering log file ingestion events
// renaming columns to match with AWSCloudTrail schema
| project-rename TrailARN=Records_responseElements_trailARN_s, AccessKeyAccessKeyId=Records_responseElements_accessKey_accessKeyId_s, AccessKeyCreateDate=Records_responseElements_accessKey_createDate_s,
AccessKeyId=Records_requestParameters_accessKeyId_s, AccessKeyStatus=Records_responseElements_accessKey_status_s, AccessKeyUserName=Records_responseElements_accessKey_userName_s,
Account=Records_responseElements_account_s, AccountAttributeNameSetItems=Records_requestParameters_accountAttributeNameSet_items_s, Acl=Records_requestParameters_acl_s,
AmazonProvidedIpv6CidrBlock=Records_requestParameters_amazonProvidedIpv6CidrBlock_b, Arn=Records_responseElements_arn_s, AssociationIdRequestParameter=Records_requestParameters_associationId_s,
AssociationIdResponseElement=Records_responseElements_associationId_s, AssumedRoleUserArn=Records_responseElements_assumedRoleUser_arn_s, AssumedRoleUserAssumedRoleId=Records_responseElements_assumedRoleUser_assumedRoleId_s,
AssumeRolePolicyDocument=Records_requestParameters_assumeRolePolicyDocument_s, Attribute=Records_requestParameters_attribute_s, AuthenticationMethod=Records_additionalEventData_AuthenticationMethod_s,
AvailabilityZone=Records_requestParameters_availabilityZone_s, AwsRegion=Records_awsRegion_s, AWSServiceName=Records_requestParameters_aWSServiceName_s, BlockDeviceMappingItems=Records_requestParameters_blockDeviceMapping_items_s,
BucketName=Records_requestParameters_bucketName_s, BucketPolicy_Version=Records_requestParameters_bucketPolicy_Version_s, BucketPolicytatement=Records_requestParameters_bucketPolicy_Statement_s,
BytesTransferredIn=Records_additionalEventData_bytesTransferredIn_d, BytesTransferredOut=Records_additionalEventData_bytesTransferredOut_d, CidrBlock=Records_requestParameters_cidrBlock_s, CipherSuite=Records_additionalEventData_CipherSuite_s,
ClientToken=Records_requestParameters_clientToken_g, ConsoleLogin=Records_responseElements_ConsoleLogin_s, Cors=Records_requestParameters_cors_s, CredentialsAccessKeyId=Records_responseElements_credentials_accessKeyId_s,
CredentialsExpiration=Records_responseElements_credentials_expiration_s, CredentialsSessionToken=Records_responseElements_credentials_sessionToken_s,
DescribeInstanceCreditSpecificationsRequestInstanceIdContent=Records_requestParameters_DescribeInstanceCreditSpecificationsRequest_InstanceId_content_s,
DescribeInstanceCreditSpecificationsRequestInstanceIdTag=Records_requestParameters_DescribeInstanceCreditSpecificationsRequest_InstanceId_tag_d,
DescribeVpcClassicLinkDnsSupportRequestVpcIdsContent=Records_requestParameters_DescribeVpcClassicLinkDnsSupportRequest_VpcIds_content_s,
DescribeVpcClassicLinkDnsSupportRequestVpcIdsTag=Records_requestParameters_DescribeVpcClassicLinkDnsSupportRequest_VpcIds_tag_d, Description=Records_requestParameters_description_s,
DestinationCidrBlock=Records_requestParameters_destinationCidrBlock_s, DetctorId=Records_responseElements_detectorId_g, DetectorId=Records_requestParameters_detectorId_g, DisableApiTermination=Records_requestParameters_disableApiTermination_b,
DurationSeconds=Records_requestParameters_durationSeconds_d, EbsOptimized=Records_requestParameters_ebsOptimized_b, EventID_g=Records_eventID_g, Enable=Records_requestParameters_enable_b,
EnableLogFileValidation=Records_requestParameters_enableLogFileValidation_b, EncodingType=Records_requestParameters_encoding_type_s, Encryption=Records_requestParameters_encryption_s, EndTime=Records_requestParameters_endTime_s,
ErrorCode=Records_errorCode_s, ErrorMessage=Records_errorMessage_s, EventName=Records_eventName_s, EventSelector=Records_responseElements_eventSelectors_s, EventSelectors=Records_requestParameters_eventSelectors_s,
EventSource=Records_eventSource_s, EventTime=Records_eventTime_t, EventType=Records_eventType_s, EventVersion=Records_eventVersion_s, ExternalId=Records_requestParameters_externalId_g, FilterSet_items=Records_requestParameters_filterSet_items_s,
FindingCriteria_criterionervice_archived=Records_requestParameters_findingCriteria_criterion_service_archived_s, FindingIds=Records_requestParameters_findingIds_s, FindingStatisticTypes=Records_requestParameters_findingStatisticTypes_s,
GatewayId=Records_requestParameters_gatewayId_s, GroupDescription=Records_requestParameters_groupDescription_s, GroupIdRequestParameter=Records_requestParameters_groupId_s, GroupIdResponseElement=Records_responseElements_groupId_s,
GroupName=Records_requestParameters_groupName_s, Host=Records_requestParameters_Host_s, HostRequestParameter=Records_requestParameters_host_s, IamInstanceProfileName=Records_requestParameters_iamInstanceProfile_name_s,
ImagesSetItems=Records_requestParameters_imagesSet_items_s, IncludeGlobalServiceEventsRequestParameter=Records_requestParameters_includeGlobalServiceEvents_b, IncludeGlobalServiceEventsResponseElement=Records_responseElements_includeGlobalServiceEvents_b,
IncludeShadowTrails=Records_requestParameters_includeShadowTrails_b, InstanceId=Records_requestParameters_instanceId_s, InstanceProfileArn=Records_responseElements_instanceProfile_arn_s,
InstanceProfileCreateDate=Records_responseElements_instanceProfile_createDate_s, InstanceProfileInstanceProfileId=Records_responseElements_instanceProfile_instanceProfileId_s,
InstanceProfileInstanceProfileName=Records_responseElements_instanceProfile_instanceProfileName_s, InstanceProfileName=Records_requestParameters_instanceProfileName_s, InstanceProfilePath=Records_responseElements_instanceProfile_path_s,
InstanceProfileRoles=Records_responseElements_instanceProfile_roles_s, InstanceSetItems=Records_responseElements_instancesSet_items_s, InstancesSet_items=Records_requestParameters_instancesSet_items_s, InstanceTenancy=Records_requestParameters_instanceTenancy_s,
InstanceType=Records_requestParameters_instanceType_s, InternetGatewayId=Records_requestParameters_internetGatewayId_s, InternetGatewayIdSet_items=Records_requestParameters_internetGatewayIdSet_items_s,
InternetGatewayInternetGatewayId=Records_responseElements_internetGateway_internetGatewayId_s, IpPermissions_items=Records_requestParameters_ipPermissions_items_s, IsMultiRegionTrailRequestParameter=Records_requestParameters_isMultiRegionTrail_b,
IsMultiRegionTrailResponseElement=Records_responseElements_isMultiRegionTrail_b, IsOrganizationTrailRequestParameter=Records_requestParameters_isOrganizationTrail_b, IsOrganizationTrailResponseElement=Records_responseElements_isOrganizationTrail_b,
Key=Records_requestParameters_key_s, KeyFingerprint=Records_responseElements_keyFingerprint_s, KeyNameRequestParameter=Records_requestParameters_keyName_s, KeyNameResponseElement=Records_responseElements_keyName_s, KeySet_items=Records_requestParameters_keySet_items_s,
KmsKeyId=Records_requestParameters_kmsKeyId_s, Lifecycle=Records_requestParameters_lifecycle_s, List_type=Records_requestParameters_list_type_s, Location=Records_requestParameters_location_s, LogFileValidationEnabled=Records_responseElements_logFileValidationEnabled_b,
Logging=Records_requestParameters_logging_s, LoginTo=Records_additionalEventData_LoginTo_s, LookupAttributes=Records_requestParameters_lookupAttributes_s, Marker=Records_requestParameters_marker_s, MaxItems=Records_requestParameters_maxItems_d,
MaxResults=Records_requestParameters_maxResults_s, MaxResults_d=Records_requestParameters_maxResults_d, MaxSessionDuration=Records_requestParameters_maxSessionDuration_d, MFAUsed=Records_additionalEventData_MFAUsed_s,
MobileVersion=Records_additionalEventData_MobileVersion_s, Monitoring_enabled=Records_requestParameters_monitoring_enabled_b, NameRequestParameter=Records_requestParameters_name_s, NameResponseElement=Records_responseElements_name_s,
NetworkInterfaceSetItems=Records_requestParameters_networkInterfaceSet_items_s, NextToken=Records_requestParameters_nextToken_s, Object_lock=Records_requestParameters_object_lock_s,
OwnerId=Records_responseElements_ownerId_s, Path=Records_requestParameters_path_s, Policy=Records_requestParameters_policy_s, PolicyArn=Records_requestParameters_policyArn_s, PolicyStatus=Records_requestParameters_policyStatus_s,
Prefix=Records_requestParameters_prefix_s, PublicAccessBlock=Records_requestParameters_publicAccessBlock_s, PublicKeyMaterial=Records_requestParameters_publicKeyMaterial_s, ReadOnly=Records_readOnly_b, RecipientAccountId=Records_recipientAccountId_s,
Replication=Records_requestParameters_replication_s, RequestID=Records_requestID_s, RequestIdResponseElement=Records_responseElements_requestId_g, RequestID_g=Records_requestID_g, requestParameters_enableDnsHostnames_value=Records_requestParameters_enableDnsHostnames_value_b,
RequestPayment=Records_requestParameters_requestPayment_s, ReservationId=Records_responseElements_reservationId_s, ResourceIdList=Records_requestParameters_resourceIdList_s, Resources=Records_resources_s, ResourcesSet_items=Records_requestParameters_resourcesSet_items_s,
Return=Records_responseElements__return_b, RoleArnRequestParameter=Records_requestParameters_roleArn_s, RoleArnResponseElement=Records_responseElements_role_arn_s, RoleAssumeRolePolicyDocument=Records_responseElements_role_assumeRolePolicyDocument_s,
RoleCreateDate=Records_responseElements_role_createDate_s, RoleName=Records_requestParameters_roleName_s, RolePath=Records_responseElements_role_path_s, RoleRoleId=Records_responseElements_role_roleId_s, RoleRoleName=Records_responseElements_role_roleName_s,
RoleSessionName=Records_requestParameters_roleSessionName_s, RoleSessionName_g=Records_requestParameters_roleSessionName_g, RoleTags=Records_responseElements_role_tags_s, RouteTableId=Records_requestParameters_routeTableId_s,
RouteTableIdSet_items=Records_requestParameters_routeTableIdSet_items_s, RouteTableOwnerId=Records_responseElements_routeTable_ownerId_s, RouteTableRouteSetItems=Records_responseElements_routeTable_routeSet_items_s,
RouteTableRouteTableId=Records_responseElements_routeTable_routeTableId_s, RouteTableVpcId=Records_responseElements_routeTable_vpcId_s, S3BucketNameRequestParameter=Records_requestParameters_s3BucketName_s, S3BucketNameResponseElement=Records_responseElements_s3BucketName_s,
SecurityGroupIdSetItems=Records_requestParameters_securityGroupIdSet_items_s, SessionCreationDate=Records_userIdentity_sessionContext_attributes_creationDate_t, SessionIssuerAccountId=Records_userIdentity_sessionContext_sessionIssuer_accountId_s,
SessionIssuerArn=Records_userIdentity_sessionContext_sessionIssuer_arn_s, SessionIssuerPrincipalId=Records_userIdentity_sessionContext_sessionIssuer_principalId_s, SessionIssuerype=Records_userIdentity_sessionContext_sessionIssuer_type_s,
SessionMfaAuthenticated=Records_userIdentity_sessionContext_attributes_mfaAuthenticated_s, SessionUserName=Records_userIdentity_sessionContext_sessionIssuer_userName_s, SharedEventID=Records_sharedEventID_g, SignatureVersion=Records_additionalEventData_SignatureVersion_s,
SortCriteriaAttributeName=Records_requestParameters_sortCriteria_attributeName_s, SortCriteriaOrderBy=Records_requestParameters_sortCriteria_orderBy_s, SourceIPAddress=Records_sourceIPAddress_s, SSEApplied=Records_additionalEventData_SSEApplied_s,
StartTime=Records_requestParameters_startTime_s, SubnetAssignIpv6AddressOnCreation =Records_responseElements_subnet_assignIpv6AddressOnCreation_b, SubnetAvailabilityZone =Records_responseElements_subnet_availabilityZone_s,
SubnetAvailabilityZoneId =Records_responseElements_subnet_availabilityZoneId_s, SubnetAvailableIpAddressCount =Records_responseElements_subnet_availableIpAddressCount_d, SubnetCidrBlock =Records_responseElements_subnet_cidrBlock_s,
SubnetDefaultForAz =Records_responseElements_subnet_defaultForAz_b, SubnetId=Records_requestParameters_subnetId_s, SubnetmapPublicIpOnLaunch =Records_responseElements_subnet_mapPublicIpOnLaunch_b, SubnetOwnerId =Records_responseElements_subnet_ownerId_s,
SubnetSetItems=Records_requestParameters_subnetSet_items_s, SubnetState =Records_responseElements_subnet_state_s, SubnetSubnetArn =Records_responseElements_subnet_subnetArn_s, SubnetsubnetId =Records_responseElements_subnet_subnetId_s,
SubnetvpcId =Records_responseElements_subnet_vpcId_s, Tagging=Records_requestParameters_tagging_s, TaggingTagSetTag=Records_requestParameters_Tagging_TagSet_Tag_s, TaggingXmlns=Records_requestParameters_Tagging_xmlns_s, Tags=Records_requestParameters_tags_s,
TagSetItems=Records_requestParameters_tagSet_items_s, TagSpecificationSetItems=Records_requestParameters_tagSpecificationSet_items_s, TrailName=Records_requestParameters_trailName_s, TrailNameList=Records_requestParameters_trailNameList_s,
UserAgent=Records_userAgent_s, UserArn=Records_responseElements_user_arn_s, UserCreateDate=Records_responseElements_user_createDate_s, UserData=Records_requestParameters_userData_s, UserId=Records_responseElements_userId_s, UserIdentityAccessKeyId=Records_userIdentity_accessKeyId_s,
UserIdentityAccountId=Records_userIdentity_accountId_s, UserIdentityArn=Records_userIdentity_arn_s, UserIdentityInvokedBy=Records_userIdentity_invokedBy_s, UserIdentityPrincipalId=Records_userIdentity_principalId_s, UserIdentityType=Records_userIdentity_type_s,
UserIdentityUserName=Records_userIdentity_userName_s, UserNameRequestParameter=Records_requestParameters_userName_s, UserPath=Records_responseElements_user_path_s, UserUserId=Records_responseElements_user_userId_s, UserNameResponseElement =Records_responseElements_user_userName_s,
VersionId=Records_requestParameters_versionId_s, Versioning=Records_requestParameters_versioning_s, VolumeSetItems=Records_requestParameters_volumeSet_items_s, VpcCidrBlock=Records_responseElements_vpc_cidrBlock_s,
VpcCidrBlockAssociationSetItems=Records_responseElements_vpc_cidrBlockAssociationSet_items_s, VpcDhcpOptionsId=Records_responseElements_vpc_dhcpOptionsId_s, VpcEndpointIdAdditionalEventData=Records_additionalEventData_vpcEndpointId_s, VpcEndpointId=Records_vpcEndpointId_s,
VpcId=Records_requestParameters_vpcId_s, VpcInstanceTenancy=Records_responseElements_vpc_instanceTenancy_s, VpcIsDefault=Records_responseElements_vpc_isDefault_b, VpcOwnerId=Records_responseElements_vpc_ownerId_s, VpcSetItem=Records_requestParameters_vpcSet_item_s,
VpcSetItems=Records_requestParameters_vpcSet_items_s, VpcState=Records_responseElements_vpc_state_s, VpcVpcId=Records_responseElements_vpc_vpcId_s, Website=Records_requestParameters_website_s, XAmzAcl=Records_requestParameters_x_amz_acl_s, XAmzId=Records_additionalEventData_x_amz_id_2_s,
XAmzServerSideEncryptionRequestParameter=Records_requestParameters_x_amz_server_side_encryption_s, XAmzServerSideEncryptionResponseElement=Records_responseElements_x_amz_server_side_encryption_s
| extend EventTime = todatetime(EventTime) // Column Datatype change to datetime
//Adding explicit project so that intellisense prompts for new field names
| project EventTime, EventSource,EventType, EventName, AwsRegion, SourceIPAddress, UserAgent, UserIdentityType, UserIdentityPrincipalId, UserIdentityArn, UserIdentityAccountId, UserIdentityAccessKeyId, UserIdentityUserName, S3BucketNameRequestParameter, TrailARN,
AccessKeyAccessKeyId, AccessKeyCreateDate, AccessKeyId, AccessKeyStatus, AccessKeyUserName, Account, AccountAttributeNameSetItems, Acl, AmazonProvidedIpv6CidrBlock, Arn, AssociationIdRequestParameter, AssociationIdResponseElement, AssumedRoleUserArn, AssumedRoleUserAssumedRoleId,
AssumeRolePolicyDocument, Attribute, AuthenticationMethod, AvailabilityZone, AWSServiceName, BlockDeviceMappingItems, BucketName, BucketPolicy_Version, BucketPolicytatement, BytesTransferredIn, BytesTransferredOut, CidrBlock, CipherSuite, ClientToken, ConsoleLogin, Cors,
CredentialsAccessKeyId, CredentialsExpiration, CredentialsSessionToken, DescribeInstanceCreditSpecificationsRequestInstanceIdContent, DescribeInstanceCreditSpecificationsRequestInstanceIdTag, DescribeVpcClassicLinkDnsSupportRequestVpcIdsContent, DescribeVpcClassicLinkDnsSupportRequestVpcIdsTag,
Description, DestinationCidrBlock, DetctorId, DetectorId, DisableApiTermination, DurationSeconds, EbsOptimized, EventID_g, Enable, EnableLogFileValidation, EncodingType, Encryption, EndTime, ErrorCode, ErrorMessage, EventSelector, EventSelectors, EventVersion, ExternalId, FilterSet_items,
FindingCriteria_criterionervice_archived, FindingIds, FindingStatisticTypes, GatewayId, GroupDescription, GroupIdRequestParameter, GroupIdResponseElement, GroupName, Host, HostRequestParameter, IamInstanceProfileName, ImagesSetItems, IncludeGlobalServiceEventsRequestParameter,
IncludeGlobalServiceEventsResponseElement, IncludeShadowTrails, InstanceId, InstanceProfileArn, InstanceProfileCreateDate, InstanceProfileInstanceProfileId, InstanceProfileInstanceProfileName, InstanceProfileName, InstanceProfilePath, InstanceProfileRoles, InstanceSetItems, InstancesSet_items,
InstanceTenancy, InstanceType, InternetGatewayId, InternetGatewayIdSet_items, InternetGatewayInternetGatewayId, IpPermissions_items, IsMultiRegionTrailRequestParameter, IsMultiRegionTrailResponseElement, IsOrganizationTrailRequestParameter, IsOrganizationTrailResponseElement, Key, KeyFingerprint,
KeyNameRequestParameter, KeyNameResponseElement, KeySet_items, KmsKeyId, Lifecycle, List_type, Location, LogFileValidationEnabled, Logging, LoginTo, LookupAttributes, Marker, MaxItems, MaxResults, MaxResults_d, MaxSessionDuration, MFAUsed, MobileVersion, Monitoring_enabled, NameRequestParameter,
NameResponseElement, NetworkInterfaceSetItems, NextToken, Object_lock, OnlyAttached, OwnerId, Path, Policy, PolicyArn, PolicyStatus, Prefix, PublicAccessBlock, PublicKeyMaterial, ReadOnly, RecipientAccountId, Replication, RequestID, RequestIdResponseElement,
RequestID_g, requestParameters_enableDnsHostnames_value, RequestPayment, ReservationId, ResourceIdList, Resources, ResourcesSet_items, Return, RoleArnRequestParameter, RoleArnResponseElement, RoleAssumeRolePolicyDocument, RoleCreateDate, RoleName, RolePath, RoleRoleId, RoleRoleName,
RoleSessionName, RoleSessionName_g, RoleTags, RouteTableId, RouteTableIdSet_items, RouteTableOwnerId, RouteTableRouteSetItems, RouteTableRouteTableId, RouteTableVpcId, S3BucketNameResponseElement, SecurityGroupIdSetItems, SessionCreationDate, SessionIssuerAccountId, SessionIssuerArn,
SessionIssuerPrincipalId, SessionIssuerype, SessionMfaAuthenticated, SessionUserName, SharedEventID, SignatureVersion, SortCriteriaAttributeName, SortCriteriaOrderBy, SSEApplied, StartTime, SubnetAssignIpv6AddressOnCreation, SubnetAvailabilityZone, SubnetAvailabilityZoneId,
SubnetAvailableIpAddressCount, SubnetCidrBlock, SubnetDefaultForAz, SubnetId, SubnetmapPublicIpOnLaunch, SubnetOwnerId, SubnetSetItems, SubnetState, SubnetSubnetArn, SubnetsubnetId, SubnetvpcId, Tagging, TaggingTagSetTag, TaggingXmlns, Tags, TagSetItems, TagSpecificationSetItems,
TrailName, TrailNameList, UserArn, UserCreateDate, UserData, UserId, UserIdentityInvokedBy, UserNameRequestParameter, UserPath, UserUserId, UserNameResponseElement, VersionId, Versioning, VolumeSetItems, VpcCidrBlock, VpcCidrBlockAssociationSetItems, VpcDhcpOptionsId,
VpcEndpointIdAdditionalEventData, VpcEndpointId, VpcId, VpcInstanceTenancy, VpcIsDefault, VpcOwnerId, VpcSetItem, VpcSetItems, VpcState, VpcVpcId, Website, XAmzAcl, XAmzId, XAmzServerSideEncryptionRequestParameter, XAmzServerSideEncryptionResponseElement