[ Nov 18 18:40:36 Enabled. ]
[ Nov 18 18:40:36 Rereading configuration. ]
[ Nov 18 18:40:37 Rereading configuration. ]
[ Nov 18 18:40:42 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_453f7245-5f70-4fe8-b37e-054e56d5c1c5/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_453f7245-5f70-4fe8-b37e-054e56d5c1c5/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:40:42.74604772Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:40:42.749379015Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:40:42.749393665Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:40:42.76753237Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:40:42.890047429Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.489768862Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45754"}
{"msg":"region ea44fb3e-42c4-45e3-b6f4-286282aec8b6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.519614277Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.523420731Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.527178255Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"uri":"/crucible/0/regions","method":"POST","req_id":"cba8d660-2909-4d42-9562-29a044f60850","remote_addr":"[fd00:1122:3344:101::a]:45754","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":17377,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_453f7245-5f70-4fe8-b37e-054e56d5c1c5/crucible/regions/ea44fb3e-42c4-45e3-b6f4-286282aec8b6","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.534267194Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_453f7245-5f70-4fe8-b37e-054e56d5c1c5/crucible/regions/ea44fb3e-42c4-45e3-b6f4-286282aec8b6","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.534294244Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ea44fb3e-42c4-45e3-b6f4-286282aec8b6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ea44fb3e-42c4-45e3-b6f4-286282aec8b6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.595789388Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"region":"ea44fb3e-42c4-45e3-b6f4-286282aec8b6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ea44fb3e-42c4-45e3-b6f4-286282aec8b6\" \"--data\" \"/data/regions/ea44fb3e-42c4-45e3-b6f4-286282aec8b6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.599580573Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"region":"ea44fb3e-42c4-45e3-b6f4-286282aec8b6","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.682576678Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"uri":"/crucible/0/regions","method":"POST","req_id":"019747be-9409-4fae-9129-89f0a678bd9a","remote_addr":"[fd00:1122:3344:101::a]:45754","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":98,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.791580501Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"region":"ea44fb3e-42c4-45e3-b6f4-286282aec8b6","component":"worker"}
{"msg":"region ea44fb3e-42c4-45e3-b6f4-286282aec8b6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.791624142Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.791726684Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ea44fb3e-42c4-45e3-b6f4-286282aec8b6","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.792151903Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ea44fb3e-42c4-45e3-b6f4-286282aec8b6","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.797066689Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.812516122Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ea44fb3e-42c4-45e3-b6f4-286282aec8b6","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.815184254Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ea44fb3e-42c4-45e3-b6f4-286282aec8b6","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.815727584Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.817762774Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.817823355Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.817869096Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.828127037Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:23.832171746Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:24.330716358Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"uri":"/crucible/0/regions","method":"POST","req_id":"44388292-2019-401c-a844-6fdc817585f6","remote_addr":"[fd00:1122:3344:101::a]:45754","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:24.659455345Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"uri":"/crucible/0/regions/ea44fb3e-42c4-45e3-b6f4-286282aec8b6","method":"GET","req_id":"324d15ff-d07f-4fa8-bfd8-77da9ba640b0","remote_addr":"[fd00:1122:3344:101::a]:45754","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"path is \"/data/regions/ea44fb3e-42c4-45e3-b6f4-286282aec8b6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:24.661881733Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"datafile"}
{"msg":"dataset is oxp_453f7245-5f70-4fe8-b37e-054e56d5c1c5/crucible/regions/ea44fb3e-42c4-45e3-b6f4-286282aec8b6","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:24.674303536Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"datafile"}
{"msg":"region ea44fb3e-42c4-45e3-b6f4-286282aec8b6 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:24.685618187Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:24.68576238Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-ea44fb3e-42c4-45e3-b6f4-286282aec8b6 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:24.686836701Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:24.687704058Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"uri":"/crucible/0/regions/ea44fb3e-42c4-45e3-b6f4-286282aec8b6","method":"DELETE","req_id":"18d11fba-6ca3-4d2d-a88d-6080d849a157","remote_addr":"[fd00:1122:3344:101::a]:45754","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25858,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:24.690763748Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"uri":"/crucible/0/regions/ea44fb3e-42c4-45e3-b6f4-286282aec8b6","method":"GET","req_id":"7d601506-a164-4cd0-8974-8c371d814736","remote_addr":"[fd00:1122:3344:101::a]:45754","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":40,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:24.692590714Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_453f7245-5f70-4fe8-b37e-054e56d5c1c5/crucible/regions/ea44fb3e-42c4-45e3-b6f4-286282aec8b6\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:24.705281112Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"region":"ea44fb3e-42c4-45e3-b6f4-286282aec8b6","component":"worker"}
{"msg":"region ea44fb3e-42c4-45e3-b6f4-286282aec8b6 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:24.804906003Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-18T18:41:24.829031016Z","hostname":"oxz_crucible_81488621-5898-41d4-a112-d3e574fb2d33","pid":24915,"uri":"/crucible/0/regions/ea44fb3e-42c4-45e3-b6f4-286282aec8b6","method":"GET","req_id":"467cc3e7-3e41-4f01-b2d9-69437aaf4266","remote_addr":"[fd00:1122:3344:101::a]:45754","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":71,"response_code":"200"}