[ Jul 29 18:47:59 Enabled. ]
[ Jul 29 18:47:59 Rereading configuration. ]
[ Jul 29 18:48:01 Rereading configuration. ]
[ Jul 29 18:48:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:06.311418938Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:06.314564656Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:06.314573557Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:06.330912393Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:06.77685726Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.244369085Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:61781"}
{"msg":"region b1abc552-f587-450f-b4a4-93368064ce54 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.268302143Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.273030375Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions","method":"POST","req_id":"0905f019-a303-45ed-adaa-dfde02b51db2","remote_addr":"[fd00:1122:3344:101::a]:61781","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":19685,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.27499747Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/b1abc552-f587-450f-b4a4-93368064ce54","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.284873726Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/b1abc552-f587-450f-b4a4-93368064ce54","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.284959838Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b1abc552-f587-450f-b4a4-93368064ce54\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b1abc552-f587-450f-b4a4-93368064ce54\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.373399857Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"region":"b1abc552-f587-450f-b4a4-93368064ce54","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b1abc552-f587-450f-b4a4-93368064ce54\" \"--data\" \"/data/regions/b1abc552-f587-450f-b4a4-93368064ce54\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.375416133Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"region":"b1abc552-f587-450f-b4a4-93368064ce54","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.62465957Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions","method":"POST","req_id":"d795de96-b007-48ca-a3f8-e82287b00a94","remote_addr":"[fd00:1122:3344:101::a]:61781","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.707905163Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"region":"b1abc552-f587-450f-b4a4-93368064ce54","component":"worker"}
{"msg":"region b1abc552-f587-450f-b4a4-93368064ce54 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.707956725Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.708048027Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b1abc552-f587-450f-b4a4-93368064ce54","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.708417268Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b1abc552-f587-450f-b4a4-93368064ce54","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.719498267Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.743853927Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b1abc552-f587-450f-b4a4-93368064ce54","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.751650484Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b1abc552-f587-450f-b4a4-93368064ce54","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.752387255Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.752447557Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.752488978Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.752525579Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.757321303Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.78587885Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:58.936217816Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions","method":"POST","req_id":"ea22b351-e503-4471-9153-97aa41493e10","remote_addr":"[fd00:1122:3344:101::a]:61781","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":46,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:59.871307907Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions/b1abc552-f587-450f-b4a4-93368064ce54","method":"GET","req_id":"ea76d74d-13d2-40fd-a54a-bc77e7e1defe","remote_addr":"[fd00:1122:3344:101::a]:61781","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"path is \"/data/regions/b1abc552-f587-450f-b4a4-93368064ce54\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:59.873864828Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"dataset is oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/b1abc552-f587-450f-b4a4-93368064ce54","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:59.885267106Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"region b1abc552-f587-450f-b4a4-93368064ce54 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:59.895466851Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:59.895671347Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:59.895733829Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions/b1abc552-f587-450f-b4a4-93368064ce54","method":"DELETE","req_id":"088ef72c-55c1-4e87-8118-479ae47833db","remote_addr":"[fd00:1122:3344:101::a]:61781","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":21899,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-b1abc552-f587-450f-b4a4-93368064ce54 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:59.896643814Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:59.899213646Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions/b1abc552-f587-450f-b4a4-93368064ce54","method":"GET","req_id":"419feb67-6b0b-44d9-bdd0-03ca7aeac4a5","remote_addr":"[fd00:1122:3344:101::a]:61781","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":25,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:59.90151687Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/b1abc552-f587-450f-b4a4-93368064ce54\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:59.913027001Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"region":"b1abc552-f587-450f-b4a4-93368064ce54","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:48:59.956791103Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions/b1abc552-f587-450f-b4a4-93368064ce54","method":"GET","req_id":"887281a6-54be-461b-b868-2e68ca6fbff4","remote_addr":"[fd00:1122:3344:101::a]:61781","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"region b1abc552-f587-450f-b4a4-93368064ce54 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:00.000855863Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:00.058204724Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions/b1abc552-f587-450f-b4a4-93368064ce54","method":"GET","req_id":"a2924967-4d4f-4cf8-af30-6f8c134ba1d4","remote_addr":"[fd00:1122:3344:101::a]:61781","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":40,"response_code":"200"}
{"msg":"region 1da32147-ce63-43f1-9809-b7f5d4eb486d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.05649405Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.056648135Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.056667715Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions","method":"POST","req_id":"16550c55-97a2-49db-9e15-94ee47ed5a01","remote_addr":"[fd00:1122:3344:101::a]:61781","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":225,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.066576572Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.066606563Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1da32147-ce63-43f1-9809-b7f5d4eb486d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.130396463Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"region":"1da32147-ce63-43f1-9809-b7f5d4eb486d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1da32147-ce63-43f1-9809-b7f5d4eb486d\" \"--data\" \"/data/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.13243429Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"region":"1da32147-ce63-43f1-9809-b7f5d4eb486d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.218892974Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"region":"1da32147-ce63-43f1-9809-b7f5d4eb486d","component":"worker"}
{"msg":"region 1da32147-ce63-43f1-9809-b7f5d4eb486d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.218942895Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.219028057Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1da32147-ce63-43f1-9809-b7f5d4eb486d","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.220599751Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1da32147-ce63-43f1-9809-b7f5d4eb486d","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.224257853Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.232850933Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1da32147-ce63-43f1-9809-b7f5d4eb486d","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.236374662Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.236722441Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.236775993Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.236830324Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.236870295Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.240901458Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.245428874Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:49:02.292498168Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions","method":"POST","req_id":"6aa3e38f-3c96-46ac-a96d-08a367a2d889","remote_addr":"[fd00:1122:3344:101::a]:61781","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.519543709Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52375"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.520254959Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d","method":"GET","req_id":"c0867ce8-7180-4c59-bba2-e324ec54baa3","remote_addr":"[fd00:1122:3344:101::a]:52375","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":38,"response_code":"200"}
{"msg":"path is \"/data/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.52389051Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"dataset is oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.533503739Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"snapshot is oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d@fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.542994334Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d\", \"fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.543041225Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.555014069Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d/snapshots/fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f","method":"GET","req_id":"26e5e538-ebfc-49ea-b455-92d119fcbc67","remote_addr":"[fd00:1122:3344:101::a]:52375","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":31135,"response_code":"200"}
{"msg":"path is \"/data/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.558833616Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"dataset is oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.566125989Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"snapshot is oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d@fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.572993861Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d\", \"fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.573031872Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 1da32147-ce63-43f1-9809-b7f5d4eb486d-fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.582312981Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"applying SMF actions for region 1da32147-ce63-43f1-9809-b7f5d4eb486d running snapshot fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.582934069Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.582962119Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d/snapshots/fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f/run","method":"POST","req_id":"d84d5677-b673-4263-821d-01b92e908c56","remote_addr":"[fd00:1122:3344:101::a]:52375","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":24159,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-1da32147-ce63-43f1-9809-b7f5d4eb486d-fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.599526882Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-1da32147-ce63-43f1-9809-b7f5d4eb486d-fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.611262409Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-1da32147-ce63-43f1-9809-b7f5d4eb486d-fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.626780153Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d/.zfs/snapshot/fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.627080301Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.627130422Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.627168613Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.627205104Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.627249675Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.631517455Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.637254475Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"worker"}
{"msg":"region 1da32147-ce63-43f1-9809-b7f5d4eb486d running snapshot fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.637274775Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"path is \"/data/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.797265881Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"dataset is oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.806753806Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"datafile"}
{"msg":"snapshot is oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d@fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.815289354Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_c46e28cb-d92b-45e9-b923-27607dc89692/crucible/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d\", \"fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.815337036Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-29T18:59:49.823260947Z","hostname":"oxz_crucible_f8062b59-0b52-40b7-8418-4e8d6f261fc6","pid":23091,"uri":"/crucible/0/regions/1da32147-ce63-43f1-9809-b7f5d4eb486d/snapshots/fb35fbf1-f07c-44a6-b4e6-41d53ee81f2f/run","method":"POST","req_id":"8c1f5cde-4f63-4528-ae26-ec3e12643c0c","remote_addr":"[fd00:1122:3344:101::a]:52375","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":26021,"response_code":"200"}