[ Oct 12 03:53:12 Enabled. ]
[ Oct 12 03:53:12 Rereading configuration. ]
[ Oct 12 03:53:14 Rereading configuration. ]
[ Oct 12 03:53:16 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_c0c860af-959d-47ac-8156-a6981a984705/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_c0c860af-959d-47ac-8156-a6981a984705/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:16.292049923Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:16.300722531Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:16.300736911Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:16.310072487Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:16.497252291Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.584391608Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:64237"}
{"msg":"region 849d203c-c779-4efa-aae5-cdc5b891a2ef state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.593729074Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.594124659Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"uri":"/crucible/0/regions","method":"POST","req_id":"ef6b9b22-ab66-446b-9335-76f02bca28bb","remote_addr":"[fd00:1122:3344:101::c]:64237","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":4376,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.598387072Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_c0c860af-959d-47ac-8156-a6981a984705/crucible/regions/849d203c-c779-4efa-aae5-cdc5b891a2ef","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.608679619Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_c0c860af-959d-47ac-8156-a6981a984705/crucible/regions/849d203c-c779-4efa-aae5-cdc5b891a2ef","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.60872825Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"849d203c-c779-4efa-aae5-cdc5b891a2ef\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/849d203c-c779-4efa-aae5-cdc5b891a2ef\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.670696529Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"region":"849d203c-c779-4efa-aae5-cdc5b891a2ef","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"849d203c-c779-4efa-aae5-cdc5b891a2ef\" \"--data\" \"/data/regions/849d203c-c779-4efa-aae5-cdc5b891a2ef\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.67077285Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"region":"849d203c-c779-4efa-aae5-cdc5b891a2ef","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.938731047Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"region":"849d203c-c779-4efa-aae5-cdc5b891a2ef","component":"worker"}
{"msg":"region 849d203c-c779-4efa-aae5-cdc5b891a2ef state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.938790307Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.938887618Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-849d203c-c779-4efa-aae5-cdc5b891a2ef","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.941248048Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-849d203c-c779-4efa-aae5-cdc5b891a2ef","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.945317568Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.94709002Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"uri":"/crucible/0/regions","method":"POST","req_id":"df1b5498-5482-41b7-850b-f5c37e0ed37c","remote_addr":"[fd00:1122:3344:101::c]:64237","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.956980653Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-849d203c-c779-4efa-aae5-cdc5b891a2ef","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.960898972Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/849d203c-c779-4efa-aae5-cdc5b891a2ef","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.966281738Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.966345239Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.96638913Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.96643248Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.971088258Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.974861985Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.308615308Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"uri":"/crucible/0/regions/849d203c-c779-4efa-aae5-cdc5b891a2ef","method":"GET","req_id":"ce536689-a438-450f-83c0-ddcf046e8e3a","remote_addr":"[fd00:1122:3344:101::c]:64237","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"path is \"/data/regions/849d203c-c779-4efa-aae5-cdc5b891a2ef\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.311844678Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"datafile"}
{"msg":"dataset is oxp_c0c860af-959d-47ac-8156-a6981a984705/crucible/regions/849d203c-c779-4efa-aae5-cdc5b891a2ef","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.323369451Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"datafile"}
{"msg":"region 849d203c-c779-4efa-aae5-cdc5b891a2ef state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.333481467Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.333636598Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.33372578Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"uri":"/crucible/0/regions/849d203c-c779-4efa-aae5-cdc5b891a2ef","method":"DELETE","req_id":"9b291b9e-1db5-4aa6-810d-8778bf6a3403","remote_addr":"[fd00:1122:3344:101::c]:64237","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":22001,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-849d203c-c779-4efa-aae5-cdc5b891a2ef (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.334822903Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.336674916Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"uri":"/crucible/0/regions/849d203c-c779-4efa-aae5-cdc5b891a2ef","method":"GET","req_id":"97ba16e9-b372-4d79-bc70-f933dc68f666","remote_addr":"[fd00:1122:3344:101::c]:64237","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":38,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.339015425Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_c0c860af-959d-47ac-8156-a6981a984705/crucible/regions/849d203c-c779-4efa-aae5-cdc5b891a2ef\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.350794042Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"region":"849d203c-c779-4efa-aae5-cdc5b891a2ef","component":"worker"}
{"msg":"region 849d203c-c779-4efa-aae5-cdc5b891a2ef state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.424431876Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.460580394Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"uri":"/crucible/0/regions/849d203c-c779-4efa-aae5-cdc5b891a2ef","method":"GET","req_id":"d7b059c9-bb52-443f-a7d2-ca7e805bf804","remote_addr":"[fd00:1122:3344:101::c]:64237","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":751,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:55.931854302Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47740"}
{"msg":"region 4089f7f3-b9c1-4ecc-a893-9c4a02450d57 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:55.932051795Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:55.932621772Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"uri":"/crucible/0/regions","method":"POST","req_id":"a499d090-a423-4ffa-8a6e-e06c58e0f91f","remote_addr":"[fd00:1122:3344:101::a]:47740","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":609,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:55.932640242Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_c0c860af-959d-47ac-8156-a6981a984705/crucible/regions/4089f7f3-b9c1-4ecc-a893-9c4a02450d57","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:55.942614626Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_c0c860af-959d-47ac-8156-a6981a984705/crucible/regions/4089f7f3-b9c1-4ecc-a893-9c4a02450d57","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:55.942655556Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4089f7f3-b9c1-4ecc-a893-9c4a02450d57\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4089f7f3-b9c1-4ecc-a893-9c4a02450d57\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:55.993939923Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"region":"4089f7f3-b9c1-4ecc-a893-9c4a02450d57","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4089f7f3-b9c1-4ecc-a893-9c4a02450d57\" \"--data\" \"/data/regions/4089f7f3-b9c1-4ecc-a893-9c4a02450d57\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:55.995906918Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"region":"4089f7f3-b9c1-4ecc-a893-9c4a02450d57","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.064488469Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"region":"4089f7f3-b9c1-4ecc-a893-9c4a02450d57","component":"worker"}
{"msg":"region 4089f7f3-b9c1-4ecc-a893-9c4a02450d57 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.064517889Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.064621571Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4089f7f3-b9c1-4ecc-a893-9c4a02450d57","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.065993128Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4089f7f3-b9c1-4ecc-a893-9c4a02450d57","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.070885128Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.082198559Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4089f7f3-b9c1-4ecc-a893-9c4a02450d57","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.087885669Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4089f7f3-b9c1-4ecc-a893-9c4a02450d57","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.088109462Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.088159403Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.088197353Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.088234704Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.091818258Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.096135932Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:57:56.208876961Z","hostname":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","pid":22747,"uri":"/crucible/0/regions","method":"POST","req_id":"357f9fa8-6226-42eb-a76e-489bdc7bbc5e","remote_addr":"[fd00:1122:3344:101::a]:47740","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":66,"response_code":"200"}