Помоги правильно добавить в этот код форматы А3х3 ...

Проблема здесь не в cond и не в размерах рамки, а в том, что вы жестко задаете CanonicalMediaName одной строкой, а DWG To PDF.pc3 для пользовательских форматов может отдавать другое каноническое имя.
Для таких форматов нужно:

1. после vla-put-ConfigName обязательно делать vla-RefreshPlotDeviceInfo;
2. искать формат не только по одной строке, а по реальным размерам листа.

Ниже полный код. Я оставил вашу логику без изменений, изменил только выбор формата и добавил служебные функции.

lisp
(defun C:AddLayN2 (/ 
		 ActiveDocument 
		 Application 
		 Display 
		 DeleteLayouts 
		 FirstSheet 
		 Flag 
		 Formats 
		 i 
		 j 
		 Layout 
		 Layouts 
		 Layer 
		 ModelSpace 
		 NumberFormats 
		 PaperSpace 
		 Points 
		 MatchSheet 
		 MinPoint 
		 MaxPoint 
		 NoMatchSheet 
		 Object 
		 Point1 
		 Point2 
		 Point1x 
		 Point1y 
		 Point2x 
		 Point2y 
		 Scale 
		 Square 
		 ViewportHight 
		 ViewportWidth 
		 Viewport 
		 X 
		 Y 
		 SideL ; Длинная сторона 
		 SideS ; Короткая сторона 
		 )
  (vl-load-com) 
  (setvar "CTAB" "Model") 
  (initget 6) 
  (setq Application (vlax-get-acad-object) 
	ActiveDocument (vla-get-ActiveDocument Application) 
	ModelSpace (vla-get-ModelSpace ActiveDocument) 
	Layouts (vla-get-Layouts ActiveDocument) 
	Display (vla-get-Display (vla-get-Preferences (vlax-get-acad-object))) 
	)
  
  (while (null Object)
    (setq Object (car (entsel "Укажите объект для определения слоя с рамками")))
    )
  (setq Layer (cdr (assoc 8 (entget Object))) 
	Formats (ssget (list (cons 8 Layer))) 
	NumberFormats (sslength Formats) 
	Scale (getreal "Масштаб 1:<100>") 
	i 0 
	Points () 
	)
  (if (not Scale) (setq Scale 100))
  (repeat NumberFormats (ssname Formats i)
    (setq Format (vlax-ename->vla-object (ssname Formats i)))
    (if 
      (and (= (vla-get-ObjectName Format) "AcDbBlockReference") (= (vla-get-IsDynamicBlock Format) :vlax-true)) 
      (progn 
	(setq Points (append Points (list (GetBoundingBox_dynblock (vlax-vla-object->ename Format)))) 
	      i (1+ i))
	)
      (progn 
	(vla-GetBoundingBox (vlax-ename->vla-object (ssname Formats i)) 'MinPoint 'MaxPoint) 
	(setq Points (append Points (list (list (vlax-safearray->list MinPoint) (vlax-safearray->list MaxPoint)))) 
	      i (1+ i))
	)
      )
    )
  
  (setq i 0)
  (repeat (length Points) 
    (setq X (append X (list (caar (nth i Points)))))
    (setq Y (append Y (list (cadar (nth i Points)))))
    (setq i (1+ i))
    )
  (if 
    (> (- (MaxElement X) (MinElement X)) (- (MaxElement Y) (MinElement Y))) 
    (setq Points (vl-sort Points (function (lambda (Points1 Points2) (< (caar Points1) (caar Points2)))))) 
    (setq Points (vl-sort Points (function (lambda (Points1 Points2) (> (cadar Points1) (cadar Points2)))))) 
    )
  
  (if (= (vla-get-LayoutCreateViewport Display) :vlax-true) 
    (progn (vla-put-LayoutCreateViewport Display :vlax-false) (setq Flag T))
    )
  
  (initget 1 "Да Нет")
  (setq DeleteLayouts (getkword "Удалить существующие листы? [Да/Нет]: ")) 
  (cond 
    ((= DeleteLayouts "Да")
      (vlax-for Layout Layouts (if (/= (vla-get-Name Layout) "Model") (vla-delete Layout)))
      (initget 6)
      (setq FirstSheet (getint "Начальный номер листа:")) 
      (vla-put-Name (vla-Item Layouts 1) (itoa FirstSheet))
      )
    ((= DeleteLayouts "Нет")
      (while (= NoMatchSheet nil)
        (progn 
	 (initget 6)
	 (setq i 0 FirstSheet (getint "Начальный номер листа:") MatchSheet nil)
	 (repeat NumberFormats (if (not (null (member (itoa (+ FirstSheet i)) (layoutlist)))) (setq MatchSheet T)) (setq i (1+ i)))
	 (if (= MatchSheet T) (alert "Совпадение имён листов!") (setq NoMatchSheet T))
	 )
        )
      )
    )
  
  (setq i 0 j 0)
  (repeat NumberFormats
    (cond 
      ((= DeleteLayouts "Да")
        (if (= i 0)
	 (progn (setq Layout (vla-item Layouts 0)) (setvar "CTAB" (itoa FirstSheet)))
	 (progn (setq Layout (vla-Add Layouts (itoa (+ FirstSheet i)))) (setvar "CTAB" (itoa (+ FirstSheet i))))
	 )
        )
      ((= DeleteLayouts "Нет")
        (setq Layout (vla-Add Layouts (itoa (+ FirstSheet i)))) (setvar "CTAB" (itoa (+ FirstSheet i)))
        )
      )
    
    (if (= (getvar 'PSLTSCALE) 1) (setvar 'PSLTSCALE 0))
    (setq Point1 (car (nth j Points)) 
	  Point2 (cadr (nth j Points)) 
	  PaperSpace (vla-get-paperspace ActiveDocument) 
	  Point1x (car Point1) Point1y (cadr Point1) Point2x (car Point2) Point2y (cadr Point2) 
	  ViewportHight (/ (abs (- Point1y Point2y)) Scale) 
	  ViewportWidth (/ (abs (- Point1x Point2x)) Scale) 
	  Viewport (vla-AddPViewport PaperSpace (vlax-3d-point (list (/ ViewportWidth 2) (/ ViewportHight 2))) ViewportWidth ViewportHight)) 
    
    (vla-display Viewport :vlax-true)
    (vla-put-mspace ActiveDocument :vlax-true) 
    (vla-zoomcenter Application (vlax-3d-point (list (/ (+ Point1x Point2x) 2) (/ (+ Point1y Point2y) 2))) 1.0) 
    (vla-put-mspace ActiveDocument :vlax-false) 
    (vla-put-standardscale Viewport acVpCustomScale) 
    (vla-put-CustomScale Viewport (/ (float 1) (float Scale))) 
    (vla-put-DisplayLocked Viewport "-1") 
    
    (vla-put-StyleSheet Layout "monochrome.ctb") 
    (vla-put-PlotType Layout 5) 

    ;;; --- ВЫБОР ФОРМАТА ПО ГАБАРИТАМ ---
    (setq SideL (max ViewportWidth ViewportHight) 
          SideS (min ViewportWidth ViewportHight)) 
    
    (vla-put-ConfigName Layout "DWG To PDF.pc3")
    (vla-RefreshPlotDeviceInfo Layout)

    (cond 
      ;; A4 (210x297)
      ((and (<= SideS 215) (<= SideL 302)) 
       (if (not (SetLayoutMedia Layout
                                (list "ISO_full_bleed_A4_(210.00_x_297.00_MM)")
                                210.00 297.00))
         (vla-put-ConfigName Layout "Нет")))
      
      ;; A3 (297x420)
      ((and (<= SideS 302) (<= SideL 425)) 
       (if (not (SetLayoutMedia Layout
                                (list "ISO_full_bleed_A3_(420.00_x_297.00_MM)")
                                297.00 420.00))
         (vla-put-ConfigName Layout "Нет")))

      ;; A4x3 (297x630)
      ((and (<= SideS 302) (<= SideL 635)) 
       (if (not (SetLayoutMedia Layout
                                (list "UserDefinedMetric (297.00 x 630.00мм)"
                                      "UserDefinedMetric (630.00 x 297.00мм)"
                                      "UserDefinedMetric (297.00 x 630.00mm)"
                                      "UserDefinedMetric (630.00 x 297.00mm)")
                                297.00 630.00))
         (vla-put-ConfigName Layout "Нет")))
      
      ;; A2 (420x594)
      ((and (<= SideS 425) (<= SideL 600)) 
       (if (not (SetLayoutMedia Layout
                                (list "ISO_full_bleed_A2_(594.00_x_420.00_MM)")
                                420.00 594.00))
         (vla-put-ConfigName Layout "Нет")))

      ;; A4x4 (297x841)
      ((and (<= SideS 302) (<= SideL 846)) 
       (if (not (SetLayoutMedia Layout
                                (list "UserDefinedMetric (297.00 x 841.00мм)"
                                      "UserDefinedMetric (841.00 x 297.00мм)"
                                      "UserDefinedMetric (297.00 x 841.00mm)"
                                      "UserDefinedMetric (841.00 x 297.00mm)")
                                297.00 841.00))
         (vla-put-ConfigName Layout "Нет")))
      
      ;; A3x3 (420x891)
      ((and (<= SideS 425) (<= SideL 896)) 
       (if (not (SetLayoutMedia Layout
                                (list "UserDefinedMetric (420.00 x 891.00мм)"
                                      "UserDefinedMetric (891.00 x 420.00мм)"
                                      "UserDefinedMetric (420.00 x 891.00mm)"
                                      "UserDefinedMetric (891.00 x 420.00mm)")
                                420.00 891.00))
         (vla-put-ConfigName Layout "Нет")))

      ;; A1 (594x841)
      ((and (<= SideS 600) (<= SideL 846)) 
       (if (not (SetLayoutMedia Layout
                                (list "ISO_full_bleed_A1_(841.00_x_594.00_MM)")
                                594.00 841.00))
         (vla-put-ConfigName Layout "Нет")))

      ;; А4х5 (297x1051)
      ((and (<= SideS 302) (<= SideL 1056)) 
       (if (not (SetLayoutMedia Layout
                                (list "UserDefinedMetric (297.00 x 1051.00мм)"
                                      "UserDefinedMetric (1051.00 x 297.00мм)"
                                      "UserDefinedMetric (297.00 x 1051.00mm)"
                                      "UserDefinedMetric (1051.00 x 297.00mm)")
                                297.00 1051.00))
         (vla-put-ConfigName Layout "Нет")))

      ;; A3x4 (420x1189)
      ((and (<= SideS 425) (<= SideL 1195)) 
       (if (not (SetLayoutMedia Layout
                                (list "UserDefinedMetric (420.00 x 1189.00мм)"
                                      "UserDefinedMetric (1189.00 x 420.00мм)"
                                      "UserDefinedMetric (420.00 x 1189.00mm)"
                                      "UserDefinedMetric (1189.00 x 420.00mm)")
                                420.00 1189.00))
         (vla-put-ConfigName Layout "Нет")))

      ;; A2x3 (594x1261)
      ((and (<= SideS 600) (<= SideL 1266)) 
       (if (not (SetLayoutMedia Layout
                                (list "UserDefinedMetric (594.00 x 1261.00мм)"
                                      "UserDefinedMetric (1261.00 x 594.00мм)"
                                      "UserDefinedMetric (594.00 x 1261.00mm)"
                                      "UserDefinedMetric (1261.00 x 594.00mm)")
                                594.00 1261.00))
         (vla-put-ConfigName Layout "Нет")))

      ;; A0 (841x1189)
      ((and (<= SideS 846) (<= SideL 1195)) 
       (if (not (SetLayoutMedia Layout
                                (list "UserDefinedMetric (1189.00 x 841.00мм)"
                                      "UserDefinedMetric (841.00 x 1189.00мм)"
                                      "UserDefinedMetric (1189.00 x 841.00mm)"
                                      "UserDefinedMetric (841.00 x 1189.00mm)")
                                841.00 1189.00))
         (vla-put-ConfigName Layout "Нет")))

      ;; A2x4 (594x1682)
      ((and (<= SideS 600) (<= SideL 1688)) 
       (if (not (SetLayoutMedia Layout
                                (list "UserDefinedMetric (594.00 x 1682.00мм)"
                                      "UserDefinedMetric (1682.00 x 594.00мм)"
                                      "UserDefinedMetric (594.00 x 1682.00mm)"
                                      "UserDefinedMetric (1682.00 x 594.00mm)")
                                594.00 1682.00))
         (vla-put-ConfigName Layout "Нет")))

      (t (vla-put-ConfigName Layout "Нет"))
    )

    (if (> ViewportHight ViewportWidth) (vla-put-PlotRotation Layout 1) (vla-put-PlotRotation Layout 0)) 
    (command "_Zoom" "_All") 
    (setq i (1+ i) j (1+ j))
    )
  
  (if (= Flag T) (vla-put-LayoutCreateViewport Display :vlax-true))
  (setvar "CTAB" "Model") 
)

(defun MinElement (X /) (car (vl-sort X '<)))
(defun MaxElement (X /) (car (vl-sort X '>)))

(defun GetBoundingBox_dynblock (ent / lst ins_pt min_point max_point)
      (if (and (setq ent (vlax-ename->vla-object ent))
	     (vlax-property-available-p ent 'isdynamicblock)
	     (equal (vla-get-isdynamicblock ent) :vlax-true))
	 (progn
	   (vlax-for item (vla-item (vla-get-blocks (vla-get-activedocument (vlax-get-acad-object))) (vla-get-name ent))
	     (if (equal (vla-get-visible item) :vlax-true) (setq lst (cons item lst)))
	   )
	   (setq ins_pt (vlax-safearray->list (vlax-variant-value (vla-get-insertionpoint ent)))
	     lst (vl-remove nil (mapcar (function (lambda (x / minp maxp)
			    (if (not (vl-catch-all-error-p (vl-catch-all-apply (function (lambda () (vla-getboundingbox x 'minp 'maxp))))))
			       (list (cons "min" (vlax-safearray->list minp)) (cons "max" (vlax-safearray->list maxp)))))) lst))
	     lst (mapcar (function (lambda (mins) (mapcar (function (lambda (fun) (apply (read mins) (mapcar (function fun) 
                 (mapcar (function (lambda (pts) (cdr (assoc mins pts)))) lst))))) (list car cadr caddr)))) (list "min" "max"))
	     lst (mapcar (function (lambda (ept) (mapcar (function (lambda (coord_pt coord_line coord_ins) 
                 (+ (* coord_pt ((eval (read (strcat "vla-get-" coord_line "EffectiveScaleFactor"))) ent)) coord_ins)))
                 ept '("X" "Y" "Z") ins_pt))) lst)
	   )
	 )
      )
)

(defun VariantToList (val /)
  (cond
    ((= (type val) 'VARIANT) (VariantToList (vlax-variant-value val)))
    ((= (type val) 'SAFEARRAY) (vlax-safearray->list val))
    ((not (atom val)) val)
    (T nil)
  )
)

(defun MediaNameExists-p (Layout MediaName / MediaList)
  (setq MediaList (VariantToList (vlax-invoke Layout 'GetCanonicalMediaNames)))
  (if (member MediaName MediaList) T nil)
)

(defun FindMediaBySize (Layout Size1 Size2 / MediaList S1 S2 Item Found)
  (setq MediaList (VariantToList (vlax-invoke Layout 'GetCanonicalMediaNames))
        S1 (strcase (rtos Size1 2 2))
        S2 (strcase (rtos Size2 2 2))
        Found nil)
  (foreach Item MediaList
    (if (and (null Found)
             (vl-string-search S1 (strcase Item))
             (vl-string-search S2 (strcase Item)))
      (setq Found Item)
    )
  )
  Found
)

(defun SetLayoutMedia (Layout CandidateList Size1 Size2 / Item Found)
  (setq Found nil)
  (foreach Item CandidateList
    (if (and (null Found) (MediaNameExists-p Layout Item))
      (setq Found Item)
    )
  )
  (if (null Found)
    (setq Found (FindMediaBySize Layout Size1 Size2))
  )
  (if Found
    (progn
      (vla-put-CanonicalMediaName Layout Found)
      T
    )
    nil
  )
)

Что изменено по сути:

• добавлен vla-RefreshPlotDeviceInfo сразу после выбора DWG To PDF.pc3;
• для всех пользовательских форматов, включая A3x3 и A3x4, сделан поиск по нескольким вариантам имени и по самим размерам.

Важный момент: если после этого A3x3 все равно не назначится, значит проблема уже не в Lisp. Если PdfMediaDiag не видит размер 891 x 420, значит DWG To PDF.pc3 реально не отдает этот формат в список доступных media, даже если он визуально есть в редакторе PC3. В таком случае надо пересоздать именно этот custom paper size в DWG To PDF.pc3.

Могу еще дать отдельную маленькую команду-диагностику, которая покажет прямо из Lisp, какие CanonicalMediaName видит текущий layout у DWG To PDF.pc3.