[ Aug 27 19:02:02 Enabled. ]
[ Aug 27 19:02:02 Rereading configuration. ]
[ Aug 27 19:02:08 Rereading configuration. ]
[ Aug 27 19:02:18 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:02:18.381075795Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:02:18.381941243Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:02:18.381952203Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:02:18.397209375Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:02:18.618107853Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.700951337Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63875"}
{"msg":"region 71b32dc2-79fd-498d-a4cb-1cea79abec79 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.711794638Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.714927172Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.718992005Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"uri":"/crucible/0/regions","method":"POST","req_id":"2d2da591-0cde-4150-95aa-73108382ae8a","remote_addr":"[fd00:1122:3344:101::a]:63875","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":10114,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/71b32dc2-79fd-498d-a4cb-1cea79abec79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.72606116Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/71b32dc2-79fd-498d-a4cb-1cea79abec79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.7260799Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"71b32dc2-79fd-498d-a4cb-1cea79abec79\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/71b32dc2-79fd-498d-a4cb-1cea79abec79\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.77656056Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"region":"71b32dc2-79fd-498d-a4cb-1cea79abec79","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"71b32dc2-79fd-498d-a4cb-1cea79abec79\" \"--data\" \"/data/regions/71b32dc2-79fd-498d-a4cb-1cea79abec79\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.776627152Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"region":"71b32dc2-79fd-498d-a4cb-1cea79abec79","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.915992966Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"region":"71b32dc2-79fd-498d-a4cb-1cea79abec79","component":"worker"}
{"msg":"region 71b32dc2-79fd-498d-a4cb-1cea79abec79 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.916043877Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.916159429Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-71b32dc2-79fd-498d-a4cb-1cea79abec79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.916560588Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-71b32dc2-79fd-498d-a4cb-1cea79abec79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.921059119Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.937817211Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-71b32dc2-79fd-498d-a4cb-1cea79abec79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.940940915Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/71b32dc2-79fd-498d-a4cb-1cea79abec79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.946849136Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.948895657Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.948971129Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.94902957Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.95340826Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.959125606Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:03.988923914Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"uri":"/crucible/0/regions","method":"POST","req_id":"91a7230c-6dec-4e82-a291-425f9682af69","remote_addr":"[fd00:1122:3344:101::a]:63875","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":90,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:04.38265337Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"uri":"/crucible/0/regions/71b32dc2-79fd-498d-a4cb-1cea79abec79","method":"GET","req_id":"b5fba66c-c8b1-471e-8e72-87559574b9a4","remote_addr":"[fd00:1122:3344:101::a]:63875","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"path is \"/data/regions/71b32dc2-79fd-498d-a4cb-1cea79abec79\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:04.385984488Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"dataset is oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/71b32dc2-79fd-498d-a4cb-1cea79abec79","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:04.396087995Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"region 71b32dc2-79fd-498d-a4cb-1cea79abec79 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:04.405500757Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:04.405645539Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:04.405720571Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"uri":"/crucible/0/regions/71b32dc2-79fd-498d-a4cb-1cea79abec79","method":"DELETE","req_id":"3709717b-8f0f-4d6c-9c87-d533b227689a","remote_addr":"[fd00:1122:3344:101::a]:63875","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":19855,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-71b32dc2-79fd-498d-a4cb-1cea79abec79 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:04.406722912Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:04.408406306Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"uri":"/crucible/0/regions/71b32dc2-79fd-498d-a4cb-1cea79abec79","method":"GET","req_id":"87f5bb42-580c-49b4-ba35-7a66a23c3ed0","remote_addr":"[fd00:1122:3344:101::a]:63875","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":29,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:04.412304366Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/71b32dc2-79fd-498d-a4cb-1cea79abec79\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:04.423133366Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"region":"71b32dc2-79fd-498d-a4cb-1cea79abec79","component":"worker"}
{"msg":"region 71b32dc2-79fd-498d-a4cb-1cea79abec79 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:04.480073128Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:04.496911571Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"uri":"/crucible/0/regions/71b32dc2-79fd-498d-a4cb-1cea79abec79","method":"GET","req_id":"037adacb-c5b4-46f8-9acc-b069eda16630","remote_addr":"[fd00:1122:3344:101::a]:63875","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"region ac0eda82-3716-4aa4-8bf8-7a97c97c00e5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.836465881Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.836607554Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"uri":"/crucible/0/regions","method":"POST","req_id":"c611fc36-eca4-46d5-a4b6-3c6842489c23","remote_addr":"[fd00:1122:3344:101::a]:63875","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":212,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.836625614Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.845938594Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.845993086Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ac0eda82-3716-4aa4-8bf8-7a97c97c00e5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.896044447Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"region":"ac0eda82-3716-4aa4-8bf8-7a97c97c00e5","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ac0eda82-3716-4aa4-8bf8-7a97c97c00e5\" \"--data\" \"/data/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.896108508Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"region":"ac0eda82-3716-4aa4-8bf8-7a97c97c00e5","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.963610116Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"region":"ac0eda82-3716-4aa4-8bf8-7a97c97c00e5","component":"worker"}
{"msg":"region ac0eda82-3716-4aa4-8bf8-7a97c97c00e5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.963670327Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.963764579Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ac0eda82-3716-4aa4-8bf8-7a97c97c00e5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.965201868Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ac0eda82-3716-4aa4-8bf8-7a97c97c00e5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.968123928Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.976634042Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.978390498Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"uri":"/crucible/0/regions","method":"POST","req_id":"7fb7ae64-bade-4513-bddf-40391ae976b1","remote_addr":"[fd00:1122:3344:101::a]:63875","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ac0eda82-3716-4aa4-8bf8-7a97c97c00e5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.980125633Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.980384198Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.98043811Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.98047859Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.980517791Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.98440228Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:03:05.989350281Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.474998556Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55719"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.475122728Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"uri":"/crucible/0/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5","method":"GET","req_id":"836ad544-eeaa-4e41-8128-8edbc1dbd4ce","remote_addr":"[fd00:1122:3344:101::a]:55719","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"path is \"/data/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.522115687Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"dataset is oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.53156799Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"snapshot is oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5@53a69295-aafa-4355-b3ec-8cdd653915eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.540904671Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5\", \"53a69295-aafa-4355-b3ec-8cdd653915eb\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.540937051Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.590145746Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"uri":"/crucible/0/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5/snapshots/53a69295-aafa-4355-b3ec-8cdd653915eb","method":"GET","req_id":"bad0414e-020a-4f7d-985a-3176c7e4b339","remote_addr":"[fd00:1122:3344:101::a]:55719","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":88683,"response_code":"200"}
{"msg":"path is \"/data/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.593619717Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"dataset is oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.602068179Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"snapshot is oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5@53a69295-aafa-4355-b3ec-8cdd653915eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.610248956Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5\", \"53a69295-aafa-4355-b3ec-8cdd653915eb\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.610288917Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot ac0eda82-3716-4aa4-8bf8-7a97c97c00e5-53a69295-aafa-4355-b3ec-8cdd653915eb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.620435994Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"applying SMF actions for region ac0eda82-3716-4aa4-8bf8-7a97c97c00e5 running snapshot 53a69295-aafa-4355-b3ec-8cdd653915eb (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.621001606Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.621031826Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"uri":"/crucible/0/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5/snapshots/53a69295-aafa-4355-b3ec-8cdd653915eb/run","method":"POST","req_id":"27315bf7-2dc7-4a70-8dad-79cf12b1deb7","remote_addr":"[fd00:1122:3344:101::a]:55719","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27875,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-ac0eda82-3716-4aa4-8bf8-7a97c97c00e5-53a69295-aafa-4355-b3ec-8cdd653915eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.629446588Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-ac0eda82-3716-4aa4-8bf8-7a97c97c00e5-53a69295-aafa-4355-b3ec-8cdd653915eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.658958551Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-ac0eda82-3716-4aa4-8bf8-7a97c97c00e5-53a69295-aafa-4355-b3ec-8cdd653915eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.767379573Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5/.zfs/snapshot/53a69295-aafa-4355-b3ec-8cdd653915eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.76768308Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.767746921Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.767794922Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.767841823Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.767908204Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"path is \"/data/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.772207102Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.777631353Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"dataset is oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.780570103Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.787178668Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"worker"}
{"msg":"region ac0eda82-3716-4aa4-8bf8-7a97c97c00e5 running snapshot 53a69295-aafa-4355-b3ec-8cdd653915eb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.787222129Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"datafile"}
{"msg":"snapshot is oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5@53a69295-aafa-4355-b3ec-8cdd653915eb","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.788858372Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_a954bc02-0277-475f-b5f2-befa522b2371/crucible/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5\", \"53a69295-aafa-4355-b3ec-8cdd653915eb\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.788876472Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-27T19:08:43.79710797Z","hostname":"oxz_crucible_500d8529-7595-4c83-9854-2baf5d4b40b4","pid":25134,"uri":"/crucible/0/regions/ac0eda82-3716-4aa4-8bf8-7a97c97c00e5/snapshots/53a69295-aafa-4355-b3ec-8cdd653915eb/run","method":"POST","req_id":"9a729699-45a0-4351-afb8-14c30d316584","remote_addr":"[fd00:1122:3344:101::a]:55719","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":24925,"response_code":"200"}