[ Nov 13 05:08:00 Enabled. ]
[ Nov 13 05:08:00 Rereading configuration. ]
[ Nov 13 05:08:01 Rereading configuration. ]
[ Nov 13 05:08:05 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8511034a-a894-457f-9cbc-98e6461f9f5a/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_8511034a-a894-457f-9cbc-98e6461f9f5a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:08:05.473880748Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:08:05.477355959Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:08:05.477371599Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:08:05.501707561Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:08:05.599082463Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.048874155Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:53482"}
{"msg":"region 62bf8a98-240e-4501-8e05-a12ee41a5ed7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.096243149Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.102892337Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"uri":"/crucible/0/regions","method":"POST","req_id":"386d9cf8-9f01-4ef5-8f01-76912a9a5dcd","remote_addr":"[fd00:1122:3344:101::c]:53482","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":23389,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.107074764Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_8511034a-a894-457f-9cbc-98e6461f9f5a/crucible/regions/62bf8a98-240e-4501-8e05-a12ee41a5ed7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.120426471Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_8511034a-a894-457f-9cbc-98e6461f9f5a/crucible/regions/62bf8a98-240e-4501-8e05-a12ee41a5ed7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.120466691Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"62bf8a98-240e-4501-8e05-a12ee41a5ed7\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/62bf8a98-240e-4501-8e05-a12ee41a5ed7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.202330736Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"region":"62bf8a98-240e-4501-8e05-a12ee41a5ed7","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"62bf8a98-240e-4501-8e05-a12ee41a5ed7\" \"--data\" \"/data/regions/62bf8a98-240e-4501-8e05-a12ee41a5ed7\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.207629832Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"region":"62bf8a98-240e-4501-8e05-a12ee41a5ed7","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.340778766Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"uri":"/crucible/0/regions","method":"POST","req_id":"1a148ff8-86b0-476f-be58-11b34738b77e","remote_addr":"[fd00:1122:3344:101::c]:53482","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":113,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.413050248Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"region":"62bf8a98-240e-4501-8e05-a12ee41a5ed7","component":"worker"}
{"msg":"region 62bf8a98-240e-4501-8e05-a12ee41a5ed7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.413103308Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.413250539Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-62bf8a98-240e-4501-8e05-a12ee41a5ed7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.413841405Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-62bf8a98-240e-4501-8e05-a12ee41a5ed7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.421119158Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.440639749Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-62bf8a98-240e-4501-8e05-a12ee41a5ed7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.444601424Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/62bf8a98-240e-4501-8e05-a12ee41a5ed7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.445692813Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.448623619Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.448697189Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.44874196Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.46022606Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.466114952Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:01.804146047Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"uri":"/crucible/0/regions","method":"POST","req_id":"f29ee2e4-cd9b-4fcf-948f-34c60aedbf63","remote_addr":"[fd00:1122:3344:101::c]:53482","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:02.423385129Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"uri":"/crucible/0/regions/62bf8a98-240e-4501-8e05-a12ee41a5ed7","method":"GET","req_id":"bdb977c1-a385-4a43-8ea3-2eb40e70f2d8","remote_addr":"[fd00:1122:3344:101::c]:53482","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":117,"response_code":"200"}
{"msg":"path is \"/data/regions/62bf8a98-240e-4501-8e05-a12ee41a5ed7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:02.428391533Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"datafile"}
{"msg":"dataset is oxp_8511034a-a894-457f-9cbc-98e6461f9f5a/crucible/regions/62bf8a98-240e-4501-8e05-a12ee41a5ed7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:02.443434534Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"datafile"}
{"msg":"region 62bf8a98-240e-4501-8e05-a12ee41a5ed7 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:02.457684469Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:02.457868721Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-62bf8a98-240e-4501-8e05-a12ee41a5ed7 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:02.459363483Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:02.460767306Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"uri":"/crucible/0/regions/62bf8a98-240e-4501-8e05-a12ee41a5ed7","method":"DELETE","req_id":"1aa1605d-9e3d-45d0-865a-8352a51465aa","remote_addr":"[fd00:1122:3344:101::c]:53482","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":32423,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:02.464446548Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"uri":"/crucible/0/regions/62bf8a98-240e-4501-8e05-a12ee41a5ed7","method":"GET","req_id":"9827f928-b569-4bdf-a53b-afb86130cf06","remote_addr":"[fd00:1122:3344:101::c]:53482","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":39,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:02.464834901Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8511034a-a894-457f-9cbc-98e6461f9f5a/crucible/regions/62bf8a98-240e-4501-8e05-a12ee41a5ed7\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:02.47948849Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"region":"62bf8a98-240e-4501-8e05-a12ee41a5ed7","component":"worker"}
{"msg":"region 62bf8a98-240e-4501-8e05-a12ee41a5ed7 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:02.553110773Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T05:09:02.554936319Z","hostname":"oxz_crucible_4134cecf-46e8-4040-95d2-8570b8a2f208","pid":22483,"uri":"/crucible/0/regions/62bf8a98-240e-4501-8e05-a12ee41a5ed7","method":"GET","req_id":"edfbd26e-0345-431f-965c-105c89bdf10e","remote_addr":"[fd00:1122:3344:101::c]:53482","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":109,"response_code":"200"}