[ Feb 11 21:30:30 Enabled. ]
[ Feb 11 21:30:30 Rereading configuration. ]
[ Feb 11 21:30:31 Rereading configuration. ]
[ Feb 11 21:30:35 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d6b0e627-0943-464e-beff-e4b5d0dec2d1/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d6b0e627-0943-464e-beff-e4b5d0dec2d1/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:30:35.329697941Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:30:35.33369143Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:30:35.33370277Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:30:35.360700257Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:30:35.522611698Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.130412609Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:53463"}
{"msg":"region b5498875-4e7f-4e55-80f0-dab0752460b0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.168548472Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.173696649Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.173710439Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"uri":"/crucible/0/regions","method":"POST","req_id":"fafeab1d-8716-4ae7-ac3a-a82809081ed0","remote_addr":"[fd00:1122:3344:101::b]:53463","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":16165,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_d6b0e627-0943-464e-beff-e4b5d0dec2d1/crucible/regions/b5498875-4e7f-4e55-80f0-dab0752460b0","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.183103681Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d6b0e627-0943-464e-beff-e4b5d0dec2d1/crucible/regions/b5498875-4e7f-4e55-80f0-dab0752460b0","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.183131422Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b5498875-4e7f-4e55-80f0-dab0752460b0\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b5498875-4e7f-4e55-80f0-dab0752460b0\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.233137778Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"region":"b5498875-4e7f-4e55-80f0-dab0752460b0","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b5498875-4e7f-4e55-80f0-dab0752460b0\" \"--data\" \"/data/regions/b5498875-4e7f-4e55-80f0-dab0752460b0\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.235418994Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"region":"b5498875-4e7f-4e55-80f0-dab0752460b0","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.54682888Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"uri":"/crucible/0/regions","method":"POST","req_id":"9e913511-4806-4a09-89b6-98e7f7e1dd64","remote_addr":"[fd00:1122:3344:101::b]:53463","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.549535147Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"region":"b5498875-4e7f-4e55-80f0-dab0752460b0","component":"worker"}
{"msg":"region b5498875-4e7f-4e55-80f0-dab0752460b0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.549550988Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.549704021Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b5498875-4e7f-4e55-80f0-dab0752460b0","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.550119142Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b5498875-4e7f-4e55-80f0-dab0752460b0","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.558327532Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.572014633Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b5498875-4e7f-4e55-80f0-dab0752460b0","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.57555567Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b5498875-4e7f-4e55-80f0-dab0752460b0","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.576135925Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.578746019Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.578804791Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.578844612Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.593912614Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.598637701Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:29.856593316Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"uri":"/crucible/0/regions","method":"POST","req_id":"f8a0cd8e-6e92-486d-aa49-23147855cc42","remote_addr":"[fd00:1122:3344:101::b]:53463","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":52,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:54.424368405Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"uri":"/crucible/0/regions/b5498875-4e7f-4e55-80f0-dab0752460b0","method":"GET","req_id":"8a7a388a-b1f5-4447-bf5e-d6f36e3d03ad","remote_addr":"[fd00:1122:3344:101::b]:53463","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"path is \"/data/regions/b5498875-4e7f-4e55-80f0-dab0752460b0\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:54.431023269Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"datafile"}
{"msg":"dataset is oxp_d6b0e627-0943-464e-beff-e4b5d0dec2d1/crucible/regions/b5498875-4e7f-4e55-80f0-dab0752460b0","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:54.442856862Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"datafile"}
{"msg":"region b5498875-4e7f-4e55-80f0-dab0752460b0 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:54.452364967Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:54.452533861Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-b5498875-4e7f-4e55-80f0-dab0752460b0 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:54.453536726Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:54.455016962Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"uri":"/crucible/0/regions/b5498875-4e7f-4e55-80f0-dab0752460b0","method":"DELETE","req_id":"356dfdb2-fa76-4bfc-b190-0497d1c2c7fd","remote_addr":"[fd00:1122:3344:101::b]:53463","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":24042,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:54.457672008Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"uri":"/crucible/0/regions/b5498875-4e7f-4e55-80f0-dab0752460b0","method":"GET","req_id":"82bf8855-0464-4435-8499-36ffa669df88","remote_addr":"[fd00:1122:3344:101::b]:53463","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:54.458356425Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d6b0e627-0943-464e-beff-e4b5d0dec2d1/crucible/regions/b5498875-4e7f-4e55-80f0-dab0752460b0\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:54.469955022Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"region":"b5498875-4e7f-4e55-80f0-dab0752460b0","component":"worker"}
{"msg":"region b5498875-4e7f-4e55-80f0-dab0752460b0 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:54.518938012Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-11T21:41:54.589986668Z","hostname":"oxz_crucible_234bd469-e50a-4077-9b6f-221d1a06e736","pid":26087,"uri":"/crucible/0/regions/b5498875-4e7f-4e55-80f0-dab0752460b0","method":"GET","req_id":"e6b006bf-0da5-4790-9fab-5db9846dfcaa","remote_addr":"[fd00:1122:3344:101::b]:53463","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":672,"response_code":"200"}