{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:20.70992269Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46374"} {"msg":"region c712842d-07e1-4f09-8808-120b6145df2e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:20.760081352Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:20.761284341Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"uri":"/crucible/0/regions","method":"POST","req_id":"9f248dff-d8aa-4371-a7b2-4f0c507693fe","remote_addr":"[fd00:1122:3344:101::a]:46374","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":15163,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:20.761332701Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/c712842d-07e1-4f09-8808-120b6145df2e","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:20.772587585Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/c712842d-07e1-4f09-8808-120b6145df2e","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:20.772618785Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"c712842d-07e1-4f09-8808-120b6145df2e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/c712842d-07e1-4f09-8808-120b6145df2e\"","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:20.8223014Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"region":"c712842d-07e1-4f09-8808-120b6145df2e","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:20.916085564Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"uri":"/crucible/0/regions","method":"POST","req_id":"3be372a7-cb1a-4bef-a7f8-11fabe5ff5b9","remote_addr":"[fd00:1122:3344:101::a]:46374","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":91,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.057593193Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"region":"c712842d-07e1-4f09-8808-120b6145df2e","component":"worker"} {"msg":"region c712842d-07e1-4f09-8808-120b6145df2e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.057627093Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.057763835Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-c712842d-07e1-4f09-8808-120b6145df2e","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.062400527Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c712842d-07e1-4f09-8808-120b6145df2e","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.066671822Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.081795835Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c712842d-07e1-4f09-8808-120b6145df2e","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.085113536Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c712842d-07e1-4f09-8808-120b6145df2e","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.085847688Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.085908338Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.085950019Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.08598901Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.090436818Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.099800212Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:16:21.197760691Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"uri":"/crucible/0/regions","method":"POST","req_id":"2a6f0fb1-43ac-4106-a771-adb46afbf1dd","remote_addr":"[fd00:1122:3344:101::a]:46374","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":65,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.160221544Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:53462"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.165051228Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"uri":"/crucible/0/regions/c712842d-07e1-4f09-8808-120b6145df2e","method":"GET","req_id":"2569f531-afaa-4534-89c2-d79013261f39","remote_addr":"[fd00:1122:3344:101::a]:53462","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":4802,"response_code":"200"} {"msg":"path is \"/data/regions/c712842d-07e1-4f09-8808-120b6145df2e\"","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.165255391Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"datafile"} {"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/c712842d-07e1-4f09-8808-120b6145df2e","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.174616976Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"datafile"} {"msg":"snapshot is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/c712842d-07e1-4f09-8808-120b6145df2e@8b254072-f4b4-466e-b1a6-8c3feab28c95","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.183449061Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/c712842d-07e1-4f09-8808-120b6145df2e\", \"8b254072-f4b4-466e-b1a6-8c3feab28c95\"]","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.183488752Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.198033386Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"uri":"/crucible/0/regions/c712842d-07e1-4f09-8808-120b6145df2e/snapshots/8b254072-f4b4-466e-b1a6-8c3feab28c95","method":"GET","req_id":"d1090709-404a-4a08-8062-10f3d3de33f6","remote_addr":"[fd00:1122:3344:101::a]:53462","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":32804,"response_code":"200"} {"msg":"path is \"/data/regions/c712842d-07e1-4f09-8808-120b6145df2e\"","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.1982752Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"datafile"} {"msg":"dataset is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/c712842d-07e1-4f09-8808-120b6145df2e","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.206965074Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"datafile"} {"msg":"snapshot is oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/c712842d-07e1-4f09-8808-120b6145df2e@8b254072-f4b4-466e-b1a6-8c3feab28c95","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.215341123Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/c712842d-07e1-4f09-8808-120b6145df2e\", \"8b254072-f4b4-466e-b1a6-8c3feab28c95\"]","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.215370503Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot c712842d-07e1-4f09-8808-120b6145df2e-8b254072-f4b4-466e-b1a6-8c3feab28c95 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.225060662Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"datafile"} {"msg":"applying SMF actions for region c712842d-07e1-4f09-8808-120b6145df2e running snapshot 8b254072-f4b4-466e-b1a6-8c3feab28c95 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.225837634Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.225858884Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"uri":"/crucible/0/regions/c712842d-07e1-4f09-8808-120b6145df2e/snapshots/8b254072-f4b4-466e-b1a6-8c3feab28c95/run","method":"POST","req_id":"c78a7fc7-beeb-4219-9236-9bd48f333eba","remote_addr":"[fd00:1122:3344:101::a]:53462","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":27620,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-c712842d-07e1-4f09-8808-120b6145df2e-8b254072-f4b4-466e-b1a6-8c3feab28c95","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.239107378Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-c712842d-07e1-4f09-8808-120b6145df2e-8b254072-f4b4-466e-b1a6-8c3feab28c95","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.24309995Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-c712842d-07e1-4f09-8808-120b6145df2e-8b254072-f4b4-466e-b1a6-8c3feab28c95","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.265139979Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c712842d-07e1-4f09-8808-120b6145df2e/.zfs/snapshot/8b254072-f4b4-466e-b1a6-8c3feab28c95","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.265403753Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.265463364Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.265509255Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.265552585Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.265602636Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.272264599Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.279594471Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"worker"} {"msg":"region c712842d-07e1-4f09-8808-120b6145df2e running snapshot 8b254072-f4b4-466e-b1a6-8c3feab28c95 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2023-12-13T05:21:45.279608361Z","hostname":"oxz_crucible_30567c02-bab2-4f33-aae7-10e756deafff","pid":21630,"component":"datafile"}