          move.w    #$0100,d0
          clr.l     d1
          move.w    #$0400,d4
          clr.l     d2
          move.w    #$1000,d3

NotReached:

          addi.b    #$10,d2
          add.w     d0,d1
          cmp.w     d1,d4
          bgt.s     NotReached
          sub.w     d2,d1
          subi.w    #$1000,d3
          bpl.s     NotReached
          move.w    d1,d0
          swap      d0
          move.w    d3,d0


             sep #%00100000        ;8 bit accumulator
             ldx #4                ;modifying 5 locations
    ;
    loop     lda $ab1234,x         ;load
             inc a                 ;increment
             sta $ab1234,x         ;store
             dex
             bpl loop              ;next


             phb                   ;save current data bank
             sep #%00110000        ;8 bit registers
             lda #$ab              ;target bank
             pha                   ;push it to the stack & pull it...
             plb                   ;into DB, making it the default bank
             ldx #4                ;modifying 5 locations
    ;
    loop     inc $1234,x           ;effectively INC $AB1234,X
             dex
             bpl loop              ;next
    ;
             plb                   ;restore previous bank

             sei                   ;IRQs off
         wai                   ;wait for interrupt
         lda via001            ;start of interrupt handler


    #!/bin/bash
    TARGET_DIR=("$@")
    [ "x$1" == "x" ] && TARGET_DIR=(".")
    function confirmDeletion() {
        local confirm=""
        until [ "x$confirm" == 'xy' ] || [ "x$confirm" == 'xn' ]
        do
            read -ep "    Delete [y/n]: " confirm
            confirm=$(echo "$confirm" | tr [:upper:] [:lower:])
        done
        [ "x$confirm" == 'xy' ]
    }
    function deleteWithConfirmation() {
        for file in "${@}"
        do
            if rm "$file"; then
                echo "    OK: $file"
            else
                echo "    FAIL: $file"
            fi
        done
    }
    for i in {'*~','a.out','*.o','*.gch','*nppdf32Log*'}
    do
        echo "Files matching: $i"
        FILES=()
        while read -rd '' file
        do
            FILES+=("$file")
            echo "  $file"
        done < <(find "${TARGET_DIR[@]}" -depth -iname "$i" -print0)
        if [ "x${FILES[*]}" != "x" ]; then
            if confirmDeletion; then
                deleteWithConfirmation "${FILES[@]}"
            else
                echo "   Skipping"
            fi
        fi
    done


Return the open file descriptor to the calling function via the eight bit accumulator by overwriting the appropriate register stack frame element:

             sep #%00100000        ;select 8 bit accumulator
             lda #0                ;clear...
             xba                   ;.B
             lda filedes           ;get file descriptor, ...
             rep #%00100000        ;select 16 bit accumulator &...
             sta reg_a,s           ;overwrite .C's stack copy

When the accumulator is pulled it will contain the value that was in filedes.

Flag an error by setting the carry bit in SR:

             sep #%00100000        ;select 8 bit accumulator
             lda reg_sr,s          ;stack copy of SR
             ora #%00000001        ;set carry bit &...
             sta reg_sr,s          ;rewrite

Flag a successful operation by clearing the carry bit in SR:

             sep #%00100000        ;select 8 bit accumulator
             lda reg_sr,s          ;stack copy of SR
             and #%11111110        ;clear carry bit &...
             sta reg_sr,s          ;rewrite

class PromptFormat:

    botname = "Chatbort"
    username = "User"

    def __init__(self):
        pass

    #

    def default_system_prompt(self):
        raise NotImplementedError

    def first_prompt(self):
        raise NotImplementedError

    def subs_prompt(self):
        raise NotImplementedError

    def stop_conditions(self, tokenizer):
        raise NotImplementedError

    def encoding_options(self):  # (add_bos, add_eos, encode_special_tokens)
        raise NotImplementedError

    def print_bot_name(self):
        return False

    def print_extra_newline(self):
        return False


class PromptFormat_raw(PromptFormat):

    description = "Model-agnostic mode simulating a raw chatlog"

    def __init__(self):
        super().__init__()
        pass

    def default_system_prompt(self):
        return \
            f"""This is a conversation between a helpful AI assistant named {self.botname} and a """ + \
            (f"""user named {self.username}.""" if self.username != "User" else """user.""")

    def first_prompt(self):
        return \
            f"""<|system_prompt|>\n{self.username}: <|user_prompt|>\n{self.botname}:"""

    def subs_prompt(self):
        return \
            f"""{self.username}: <|user_prompt|>\n{self.botname}:"""

    def stop_conditions(self, tokenizer):
        return \
            [self.username + ":",
             self.username[0:1] + ":",
             self.username.upper() + ":",
             self.username.lower() + ":",
             tokenizer.eos_token_id]

    def encoding_options(self):
        return False, False, False

    def print_bot_name(self):
        return True

########################################################

class PromptFormat_llama(PromptFormat):

    description = "Llama-chat, Llama2-chat and Mistral-instruct models"

    def __init__(self):
        super().__init__()
        pass

    def default_system_prompt(self):
        return \
            """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  """ + \
            """Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. """ + \
            """Please ensure that your responses are socially unbiased and positive in nature."""

    def first_prompt(self):
        return \
            """[INST] <<SYS>>\n<|system_prompt|>\n<</SYS>>\n\n<|user_prompt|> [/INST]"""

    def subs_prompt(self):
        return \
            """[INST] <|user_prompt|> [/INST]"""

    def stop_conditions(self, tokenizer):
        return \
            [tokenizer.eos_token_id]

    def encoding_options(self):
        return True, False, False

    def print_extra_newline(self):
        return True


   def build_attn_mask(self, batch_size, seq_len, past_len, input_mask, device):

        if input_mask is None and seq_len == 1: return None

        if isinstance(past_len, tuple):

            attn_masks = []

            for i in range(len(past_len[1])):

                attn_mask = torch.zeros((1, 1, seq_len, past_len[1][i] + seq_len), dtype = torch.float16, device = device)
                attn_mask_triu = torch.triu(torch.full((seq_len - 1, seq_len - 1), -65504.))
                attn_mask[:, :, : seq_len - 1, past_len[1][i] + 1: past_len[1][i] + seq_len] = attn_mask_triu

                if input_mask is not None:
                    min_mask_width = min(input_mask[i].shape[-1], seq_len + past_len[1][i])
                    input_mask_part = safe_move_tensor(input_mask[i][:, :min_mask_width], attn_mask.device)
                    input_mask_part = input_mask_part.unsqueeze(1).unsqueeze(2)
                    attn_mask[:, :, :, :min_mask_width] = torch.minimum(attn_mask[:, :, :, :min_mask_width], input_mask_part)

                attn_masks.append(attn_mask)

            return attn_masks

        else:

            attn_mask = torch.zeros((batch_size, 1, seq_len, past_len + seq_len), dtype = torch.float16, device = device)
            attn_mask_triu = torch.triu(torch.full((seq_len - 1, seq_len - 1), -65504.))
            attn_mask[:, :, : seq_len - 1, past_len + 1: past_len + seq_len] = attn_mask_triu

            if input_mask is not None:
                min_mask_width = min(input_mask.shape[-1], seq_len + past_len)
                input_mask_part = safe_move_tensor(input_mask[:, :min_mask_width], attn_mask.device)
                input_mask_part = input_mask_part.unsqueeze(1).unsqueeze(2)
                attn_mask[:, :, :, :min_mask_width] = torch.minimum(attn_mask[:, :, :, :min_mask_width], input_mask_part)

            return attn_mask


            We have to develop a Java program to calculate the sum of n natural numbers. Sum of natural number N as given as sum = 1+2+3+….+N

Examples:-
1+2+3+4+5 = 15
1+2+3+4+5+6+7+8+9+10 = 55

Procedure to develop a method to find the sum of N natural numbers in Java,

    Take the N value.
    Declare an iterator variable and initialize it with 1 because natural numbers start with 1.
    Add iterator variable value into the sum variable
    Increase the value of the iterator variable by 1
    Repeat 3 and 4 steps until the number remains greater than the iterator variable

The time complexity of this procedure is O(n).

import java.util.Scanner;

public class NaturalNumberSum {

  // method to find sum of N natural numbers
  public static int naturalNumberSum(int number){

     int i = 1; // iterator variable
     // variable to store sum value
     int sum = 0;

     // loop to repeat the process
     while (i<=number) {

        // add into sum value
        sum = sum + i;
        // increase iterator variable
        i++;
     }

     // return sum value
     return sum;
  }

  public static void main(String[] args) {

     // declare variables
     int number = 0;
     int sum = 0;

     // create Scanner class object
     Scanner scan = new Scanner(System.in);

     // read input
     System.out.print("Enter N value:: ");
     number = scan.nextInt();

     // Calculate the sum value
     sum = naturalNumberSum(number);

     // display result
     System.out.println("Sum = "+sum);

     // close Scanner class objects
     scan.close();
  }
}

The output for different test-cases:-

Enter N value:: 5
Sum = 15

Enter N value:: 10
Sum = 55

In this program, we have used a while loop to find the sum of natural numbers in Java. While loop is a pre-test loop where the expression is evaluated then only statements are executed. It uses a test expression to control the loop. Before every iteration of the loop, the test expression is evaluated.

Also See:-

    Sum of digits of a number
    The sum of even digits in a number
    Sum of odd digits in a number
    Sum of first & last digit of a number
    The Sum of Digits Until Single Digit

We can also use for loop instead of using a while loop. The for loop is also a pre-test loop, where first of all initialization expression is evaluated then the condition is checked and if the condition is true then only the statements of the for loop are executed.

public static int naturalNumberSum(int number){

     int sum = 0;

     for(int i=1; i<=number; i++)
         sum+=i;

     return sum;
 }

Or,

public static int naturalNumberSum(int number){

  int sum = 0;

  for(int i=1; ; sum+=i, i++)
    if(i>number) return sum;

}

The time complexity of all above methods are O(n).
Sum of Natural Numbers in Java without using the loop

We can also do the same work without using the loop. The formula for this operation,

Sum = n * (n+1) / 2;

Example:-
Sum of first 10 natural numbers = 10*(10+1)/2 = 10*11/2 = 5*11 = 55

It is the best way to find the sum of natural numbers. The time complexity of this method is O(1).

import java.util.Scanner;

public class NaturalNumberSum {

   // method to find sum of N natural numbers
   public static int naturalNumberSum(int number){
      return number*(number+1)/2;
   }

   public static void main(String[] args) {

      // declare variables
      int number = 0;
      int sum = 0;

      // create Scanner class object
      Scanner scan = new Scanner(System.in);

      // read input
      System.out.print("Enter N value:: ");
      number = scan.nextInt();

      // Calculate the sum value
      sum = naturalNumberSum(number);

      // display result
      System.out.println("Sum = "+sum);

      // close Scanner class objects
      scan.close();
   }
}

Using recursion

We already developed java program to find the sum of the natural number using for loop, or while loop, or without using the loop. Now we will find the same using the recursion technique. In Recursion, We divided the big problems into smaller problems or sub-problems.

Sum of N natural numbers given as 1+2+….+(n-1)+n. So, the problem can be divided as n + ( (n-1) +… + 2 + 1 )

General case for finding the sum of natural number => sum(n) = n + sum(n-1); Similarly, the base case for finding the sum of natural number => sum(0) = 0; or sum(1) = 1;

import

module jtransmissiongatetb;
	wire y;
	reg a,control;
	jtransmissiongate jtgate(y,control,a);
	initial
	begin
		$display ("RESULT\ta\ty");

		a = 0; control = 0; # 50; // Initial value is set
		if ( y === 1'bz ) // Test for inversion
			$display ("PASS  \t%d\t%d",a,y);
		else
			$display ("FAIL  \t%d\t%d",a,y);
		control = 1; # 50; // Simply change the control signal
		control = 0; # 50; // Simply change the control signal
		control = 1; # 50; // Simply change the control signal
		control = 0; # 50; // Simply change the control signal

		a = 0; control = 1; # 50; // Initial value is set
		if ( y === 0 ) // Test for inversion
			$display ("PASS  \t%d\t%d",a,y);
		else
			$display ("FAIL  \t%d\t%d",a,y);

		a = 1; control = 0; # 50; // Another value
		if ( y === 1'bz ) // Test for inversion
			$display ("PASS  \t%d\t%d",a,y);
		else
			$display ("FAIL  \t%d\t%d",a,y);
		control = 1; # 50; // Simply change the control signal
		control = 0; # 50; // Simply change the control signal
		control = 1; # 50; // Simply change the control signal
		control = 0; # 50; // Simply change the control signal

		a = 1; control = 1; # 50; // Another value
		if ( y === 1 ) // Test for inversion
			$display ("PASS  \t%d\t%d",a,y);
		else
			$display ("FAIL  \t%d\t%d",a,y);

	end
  //enabling the wave dump
  initial begin
    $dumpfile("dump.vcd"); $dumpvars;
  end
endmodule


module jtransmissiongate(y,control,a);
	output y;
	input a,control;

	wire cbar;

	assign cbar = ~control;

	nmos n1(y,a,control);
	pmos p1(y,a,cbar);
	//cmos c1(y,a,control,cbar);

endmodule


module juniversalShiftRegisterTb;
  wire [3:0] DATAOUT;
  reg clock, reset;
  reg [1:0] MODE;
  reg [3:0] DATAIN;

  juniversalShiftRegister jusr(DATAOUT, clock, reset, MODE, DATAIN);

  initial
  begin
    clock =0; MODE = 2'b00; DATAIN = 4'b0000;
    reset = 1; #10; reset = 0; #10;

    $display("RSLT\tD == DOUT");
    // Start testing Right Shift mode
    MODE = 2'b00; reset = 1; #10; reset = 0; #10;
    MODE = 2'b01; DATAIN = 4'b0011; #10;
    if ( DATAOUT === 4'b1000 ) // look at previous value of DATAOUT as well
      $display("PASS\t%p is %p with %p", DATAIN, MODE, DATAOUT);
    else
      $display("FAIL\t%p is %p with %p", DATAIN, MODE, DATAOUT);
    MODE = 2'b01; DATAIN = 4'b0011; #10;
    if ( DATAOUT === 4'b1100 ) // look at previous value of DATAOUT as well
      $display("PASS\t%p is %p with %p", DATAIN, MODE, DATAOUT);
    else
      $display("FAIL\t%p is %p with %p", DATAIN, MODE, DATAOUT);


    // Start testing Left Shift mode
    MODE = 2'b00; reset = 1; #10; reset = 0; #10;
    MODE = 2'b10; DATAIN = 4'b0111; #10;
    if ( DATAOUT === 4'b0001 ) //
      $display("PASS\t%p is %p with %p", DATAIN, MODE, DATAOUT);
    else
      $display("FAIL\t%p is %p with %p", DATAIN, MODE, DATAOUT);
    MODE = 2'b10; DATAIN = 4'b0111; #10;
    if ( DATAOUT === 4'b0011 ) //
      $display("PASS\t%p is %p with %p", DATAIN, MODE, DATAOUT);
    else
      $display("FAIL\t%p is %p with %p", DATAIN, MODE, DATAOUT);


    // Start testing parallel load mode
    MODE = 2'b00; reset = 1; #10; reset = 0; #10;
    MODE = 2'b11; DATAIN = 4'b1010; #10;
    if ( DATAOUT === 4'b1010 )
      $display("PASS\t%p is %p with %p", DATAIN, MODE, DATAOUT);
    else
      $display("FAIL\t%p is %p with %p", DATAIN, MODE, DATAOUT);

    #20;
    $finish;
  end


  //enabling the wave dump
  initial begin
    $dumpfile("dump.vcd"); $dumpvars;
  end


  always #5 clock = ~clock;

endmodule

#!/bin/bash
# use predefined variables to access passed arguments
#echo arguments to the shell
echo $1 $2 $3 ' -> echo $1 $2 $3'

# We can also store arguments from bash command line in special array
args=("$@")
#echo arguments to the shell
echo ${args[0]} ${args[1]} ${args[2]} ' -> args=("$@"); echo ${args[0]} ${args[1]} ${args[2]}'

#use $@ to print out all arguments at once
echo $@ ' -> echo $@'

# use $# variable to print out
# number of arguments passed to the bash script
echo Number of arguments passed: $# ' -> echo Number of arguments passed: $#'

Let’s try executing this script and providing three arguments.

$ ./arguments.sh Bash Scripting Tutorial

The results when we execute this script:

Bash Scripting Tutorial  -> echo $1 $2 $3
Bash Scripting Tutorial  -> args=("$@"); echo ${args[0]} ${args[1]} ${args[2]}
Bash Scripting Tutorial  -> echo $@
Number of arguments passed: 3  -> echo Number of arguments passed: $#

Executing shell commands with bash

The best way to execute a separate shell command inside of a Bash script is by creating a new subshell through the $( ) syntax. Check the example below where we echo the result of running the uname -o command.

#!/bin/bash
# use a subshell $() to execute shell command
echo $(uname -o)
# executing bash command without subshell
echo uname -o

Notice that in the final line of our script, we do not execute the uname command within a subshell, therefore the text is taken literally and output as such.

$ uname -o
GNU/LINUX
$ ./subshell.sh
GNU/LINUX
uname -o

Reading User Input

We can use the read command to read input from the user. This allows a user to interact with a Bash script and help dictate the way it proceeds. Here’s an example:

#!/bin/bash

echo -e "Hi, please type the word: \c "
read  word
echo "The word you entered is: $word"
echo -e "Can you please enter two words? "
read word1 word2
echo "Here is your input: \"$word1\" \"$word2\""
echo -e "How do you feel about bash scripting? "
# read command now stores a reply into the default build-in variable $REPLY
read
echo "You said $REPLY, I'm glad to hear that! "
echo -e "What are your favorite colours ? "
# -a makes read command to read into an array
read -a colours
echo "My favorite colours are also ${colours[0]}, ${colours[1]} and ${colours[2]}:-)"

Our Bash script asks multiple questions and then is able to repeat the information back to us through variables and arrays:

$ ./read.sh
Hi, please type the word: Linuxconfig.org
The word you entered is: Linuxconfig.org
Can you please enter two words?
Debian Linux
Here is your input: "Debian" "Linux"
How do you feel about bash scripting?
good
You said good, I'm glad to hear that!
What are your favorite colours ?
blue green black
My favorite colours are also blue, green and black:-)

Bash Trap Command

The trap command can be used in Bash scripts to catch signals sent to the script and then execute a subroutine when they occur. The script below will detect a Ctrl + C interrupt.

#!/bin/bash
# bash trap command
trap bashtrap INT
# bash clear screen command
clear;
# bash trap function is executed when CTRL-C is pressed:
# bash prints message => Executing bash trap subrutine !
bashtrap()
{
    echo "CTRL+C Detected !...executing bash trap !"
}
# for loop from 1/10 to 10/10
for a in `seq 1 10`; do
    echo "$a/10 to Exit."
    sleep 1;
done
echo "Exit Bash Trap Example!!!"

In the output below you can see that we try to Ctrl + C two times but the script continues to execute.

$ ./trap.sh
1/10 to Exit.
2/10 to Exit.
^CCTRL+C Detected !...executing bash trap !
3/10 to Exit.
4/10 to Exit.
5/10 to Exit.
6/10 to Exit.
7/10 to Exit.
^CCTRL+C Detected !...executing bash trap !
8/10 to Exit.
9/10 to Exit.
10/10 to Exit.
Exit Bash Trap Example!!!

Arrays

Bash is capable of storing values in arrays. Check the sections below for two different examples.
Declare simple bash array

This example declares an array with four elements.

#!/bin/bash
#Declare array with 4 elements
ARRAY=( 'Debian Linux' 'Redhat Linux' Ubuntu Linux )
# get number of elements in the array
ELEMENTS=${#ARRAY[@]}

# echo each element in array
# for loop
for (( i=0;i<$ELEMENTS;i++)); do
    echo ${ARRAY[${i}]}
done

Executing the script will output the elements of our array:

$ ./arrays.sh
Debian Linux
Redhat Linux
Ubuntu
Linux

Read file into bash array

Rather than filling out all of the elements of our array in the Bash script itself, we can program our script to read input and put it into an array.

#!/bin/bash
# Declare array
declare -a ARRAY
# Link filedescriptor 10 with stdin
exec 10<&0
# stdin replaced with a file supplied as a first argument
exec < $1
let count=0

while read LINE; do

    ARRAY[$count]=$LINE
    ((count++))
done

echo Number of elements: ${#ARRAY[@]}
# echo array's content
echo ${ARRAY[@]}
# restore stdin from filedescriptor 10
# and close filedescriptor 10
exec 0<&10 10<&-

Now let’s execute the script and store four elements in the array by using a file’s contents for input.

$ cat bash.txt
Bash
Scripting
Tutorial
Guide
$ ./bash-script.sh bash.txt
Number of elements: 4
Bash Scripting Tutorial Guide




	*** CompressADPCM3 ***

	; JoinCode = CompressADPCM3(Source, Length, Destination, JoinCode)
	; d0                          a0      d0      a1           d1
	;
	; This function compresses a RAW sample to a given memory. The
	; result is a 3bit ADPCM code. The destination buffer must be
	; at least (Length+7)/8*3 bytes in size.
	;
	; Function of the JoinCode: See above.

		XDEF _CompressADPCM3
_CompressADPCM3
		movem.l	d2-d4,-(sp)

		move.w	d1,d3			; d3=EstMax
		swap	d1
		move.w	d1,d2			; d2=Delta
		bne.s	c3_loop
		moveq	#5,d2

c3_loop		moveq	#0,d1			; d1=Shifter
		bsr.s	c3_byte
		lsl.b	#3,d1
		bsr.s	c3_byte
		lsl.w	#3,d1
		bsr.s	c3_byte
		lsl.w	#3,d1
		bsr.s	c3_byte
		lsl.w	#3,d1
		bsr.s	c3_byte
		lsl.l	#3,d1
		bsr.s	c3_byte
		lsl.l	#3,d1
		bsr.s	c3_byte
		lsl.l	#3,d1
		bsr.s	c3_byte
		swap	d1
		move.b	d1,(a1)+
		rol.l	#8,d1
		move.b	d1,(a1)+
		rol.l	#8,d1
		move.b	d1,(a1)+

		subq.l	#8,d0			; d0=Counter
		bhi.s	c3_loop

		move.w	d2,d0			; -> d0=JoinCode
		swap	d0
		move.w	d3,d0

		movem.l	(sp)+,d2-d4
		rts

c3_byte		move.b	(a0)+,d4
		ext.w	d4
		asl.w	#6,d4
		sub.w	d3,d4
		bpl.s	c3_positive
		or.b	#%100,d1
		neg.w	d4
c3_positive	sub.w	d2,d4
		bls.s	c3_00
		sub.w	d2,d4
		bls.s	c3_01
		sub.w	d2,d4
		bls.s	c3_10
c3_11		or.b	#%11,d1
		bra.s	c3_00
c3_10		or.b	#%10,d1
		bra.s	c3_00
c3_01		or.b	#%01,d1
c3_00		bsr.s	adaptive
		rts



		*** Adaptions-Routine ***

adaptive	; d1 = SignBit + DataBit

		move.w	d2,d4
		lsr.w	#1,d4
		btst	#1,d1
		beq.s	d3_0
d3_1		btst	#0,d1
		beq.s	d3_10
d3_11		add.w	d2,d4
		add.w	d2,d4
		add.w	d2,d4
		mulu	#$6607,d2
		bra.s	d3_sign
d3_10		add.w	d2,d4
		add.w	d2,d4
		mulu	#$4D14,d2
		bra.s	d3_sign
d3_0		btst	#0,d1
		beq.s	d3_00
d3_01		add.w	d2,d4
		mulu	#$3A9F,d2
		bra.s	d3_sign
d3_00		mulu	#$399A,d2
d3_sign		btst	#2,d1
		beq.s	d3_add
		neg.w	d4
d3_add		add.w	d4,d3
		add.l	#8192,d2
		moveq	#14,d4
		asr.l	d4,d2
		rts


		END

	mov


	mov	bp, 255

traceloop:

; Evaluate whether sample point is inside or outside the shape:
;
; ( a & ( b | c ) ) | ( b & c ) = 0 <=> voxel overlaps fractal

	push 	bx

	mov 	dx, bx
	or	dx, cx
	and	dx, ax
	and	bx, cx
	or	dx, bx

	pop	bx

; Ignore the lower bits or the fractal will be too fine to see

	shr	dx, 6
	jz	endtrace

	dec	bp
	jnz	traceloop

endtrace:

; BP is 255 - the distance we had to trace

	mov	dx, bp
	not	dl

; Plot pixel

	mov	ds:[di],dl
	inc	di






// ******************

#include <torch/extension.h>
#include <c10/cuda/CUDAGuard.h>
#include <ATen/cuda/CUDAContext.h>
#include <cuda_runtime.h>module jtransmissiongatetb;
	wire y;
	reg a,control;
	jtransmissiongate jtgate(y,control,a);
	initial
	begin
		$display ("RESULT\ta\ty");

		a = 0; control = 0; # 50; // Initial value is set
		if ( y === 1'bz ) // Test for inversion
			$display ("PASS  \t%d\t%d",a,y);
		else
			$display ("FAIL  \t%d\t%d",a,y);
		control = 1; # 50; // Simply change the control signal
		control = 0; # 50; // Simply change the control signal
		control = 1; # 50; // Simply change the control signal
		control = 0; # 50; // Simply change the control signal

		a = 0; control = 1; # 50; // Initial value is set
		if ( y === 0 ) // Test for inversion
			$display ("PASS  \t%d\t%d",a,y);
		else
			$display ("FAIL  \t%d\t%d",a,y);

		a = 1; control = 0; # 50; // Another value
		if ( y === 1'bz ) // Test for inversion
			$display ("PASS  \t%d\t%d",a,y);
		else
			$display ("FAIL  \t%d\t%d",a,y);
		control = 1; # 50; // Simply change the control signal
		control = 0; # 50; // Simply change the control signal
		control = 1; # 50; // Simply change the control signal
		control = 0; # 50; // Simply change the control signal

		a = 1; control = 1; # 50; // Another value
		if ( y === 1 ) // Test for inversion
			$display ("PASS  \t%d\t%d",a,y);
		else
			$display ("FAIL  \t%d\t%d",a,y);

	end
  //enabling the wave dump
  initial begin
    $dumpfile("dump.vcd"); $dumpvars;
  end
endmodule
#include <cuda_fp16.h>
#include <cstdint>
#include <cstdio>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>

#include "config.h"

#include "cuda/pack_tensor.cuh"
#include "cuda/quantize.cuh"
#include "cuda/q_matrix.cuh"
#include "cuda/q_attn.cuh"
#include "cuda/q_mlp.cuh"
#include "cuda/q_gemm.cuh"
#include "cuda/rms_norm.cuh"
#include "cuda/rope.cuh"
#include "cuda/cache.cuh"
#include "cuda/h_gemm.cuh"

#include "cpp/quantize_func.h"
#include "cpp/sampling.h"

#include "cpp/util.h"

// Some decluttering macros

#define TORCH_CHECK_DTYPE(__x, __dtype) TORCH_CHECK((__x).dtype() == torch::__dtype, #__x " is incorrect datatype, must be " #__dtype)
#define TORCH_CHECK_DTYPE_OPT(__x, __dtype) TORCH_CHECK((__x).device().is_meta() || (__x).dtype() == torch::__dtype, #__x " is incorrect datatype, must be " #__dtype)
#define TORCH_CHECK_SHAPES(__x, __dim_x, __y, __dim_y, __scale_y) TORCH_CHECK((__x).size(__dim_x) == (__y).size(__dim_y) * __scale_y, #__x " and " #__y " have incompatible shapes")
#define TORCH_CHECK_SHAPES_OPT(__x, __dim_x, __y, __dim_y, __scale_y) TORCH_CHECK((__x).device().is_meta() || (__x).size(__dim_x) == (__y).size(__dim_y) * __scale_y, #__x " and " #__y " have incompatible shapes")


// Packing functions

void pack_rows_4
(
    torch::Tensor input,
    torch::Tensor output
)
{
    const at::cuda::OptionalCUDAGuard device_guard(device_of(input));

    TORCH_CHECK_DTYPE(input, kShort);
    TORCH_CHECK_DTYPE(output, kInt);
    TORCH_CHECK_SHAPES(input, 0, output, 0, 1);
    TORCH_CHECK_SHAPES(input, 1, output, 1, 8);

    int rows = input.size(0);
    int columns = input.size(1);

    pack_rows_4_cuda
    (
        (uint16_t*) input.data_ptr(),
        (uint32_t*) output.data_ptr(),
        rows,
        columns
    );
}

void pack_columns
(
    torch::Tensor input,
    torch::Tensor output,
    int bits
)
{
    const at::cuda::OptionalCUDAGuard device_guard(device_of(input));

    TORCH_CHECK_DTYPE(input, kShort);
    TORCH_CHECK_DTYPE(output, kInt);
    TORCH_CHECK_SHAPES(input, 1, output, 1, 1);

    int in_rows = input.size(0);
    int columns = input.size(1);
    int out_rows = output.size(0);
    int exp_out_rows = in_rows * bits / 32;
    TORCH_CHECK(out_rows == exp_out_rows, "Wrong output shape for input and bitrate")

    pack_columns_cuda
    (
        (uint16_t*) input.data_ptr(),
        (uint32_t*) output.data_ptr(),
        in_rows,
        out_rows,
        columns,
        bits
    );
}

#include "quantize_func.h"
#include "../cuda/quantize.cuh"

void quantize_range
(
    torch::Tensor quant,
    torch::Tensor scale,
    torch::Tensor out_q,
    float qzero,
    float maxq,
    torch::Tensor hessian_inv,
    torch::Tensor weights,
    torch::Tensor error,
    int a,
    int b
)
{
    int columns = weights.size(1);
    int hcolumns = hessian_inv.size(1);

    for (int c = a; c < b; c++)
    {
        quantize_cuda
        (
            ((const float*) weights.data_ptr()) + c * columns,
            ((float*) quant.data_ptr()) + c * columns,
            (const float*) scale.data_ptr(),
            out_q.device().is_meta() ? NULL : ((uint16_t*) out_q.data_ptr()) + c * columns,
            1,
            columns,
            qzero,
            maxq
        );

        adjust_error_row_cuda
        (
            (const float*) hessian_inv.data_ptr(),
            (float*) error.data_ptr(),
            (const float*) weights.data_ptr(),
            (const float*) quant.data_ptr(),
            c,
            columns,
            hcolumns
        );

        vv_mul_sub_cuda
        (
            ((const float*) hessian_inv.data_ptr()) + c * hcolumns + c,
            ((const float*) error.data_ptr()) + c * columns,
            ((float*) weights.data_ptr()) + c * columns,
            b - c,
            columns
        );
    }

    torch::Tensor x = hessian_inv.slice(0, a, b).slice(1, b).transpose(0, 1);
    torch::Tensor y = error.slice(0, a, b);
    weights.slice(0, b).addmm_(x, y, 1.0f, -1.0f);
}

//---------------------------------------------------------------------------------------------------
//---------------------------------------------------------------------------------------------------

__forceinline__ __device__ half dot22_8_h(half2(&dq)[4], const half* a_ptr, const half g_result, const half qs_h)
{
    half2 result = {};
    const half2* a2_ptr = (const half2*)a_ptr;
    #pragma unroll
    for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
    half result_h = __hadd(__low2half(result), __high2half(result));
    return __hfma(result_h, qs_h, g_result);
}

__forceinline__ __device__ half dot22_16_h(half2(&dq)[8], const half* a_ptr, const half g_result, const half qs_h)
{
    half2 result = {};
    const half2* a2_ptr = (const half2*)a_ptr;
    #pragma unroll
    for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
    half result_h = __hadd(__low2half(result), __high2half(result));
    return __hfma(result_h, qs_h, g_result);
}

__forceinline__ __device__ half dot22_32_h(half2(&dq)[16], const half* a_ptr, const half g_result, const half qs_h)
{
    half2 result = {};
    const half2* a2_ptr = (const half2*)a_ptr;
    #pragma unroll
    for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
    half result_h = __hadd(__low2half(result), __high2half(result));
    return __hfma(result_h, qs_h, g_result);
}

name: Build Wheels

on: workflow_dispatch

jobs:
  build_wheels:
    name: ${{ matrix.os }} Python ${{ matrix.pyver }} CUDA ${{ matrix.cuda }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-20.04, windows-latest]
        pyver: ["3.8", "3.9", "3.10", "3.11"]
        cuda: ["11.7.0", "11.8.0", "12.1.1"]
    defaults:
      run:
        shell: pwsh
    env:
      CUDAVER: ${{ matrix.cuda }}
      PYVER: ${{ matrix.pyver }}

    steps:
      - name: Free Disk Space
        uses: jlumbroso/free-disk-space@v1.2.0
        if: runner.os == 'Linux'
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: false
          swap-storage: false

      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.pyver }}

      - name: Setup Mamba
        uses: conda-incubator/setup-miniconda@v2.2.0
        with:
          activate-environment: "build"
          python-version: ${{ matrix.pyver }}
          miniforge-variant: Mambaforge
          miniforge-version: latest
          use-mamba: true
          add-pip-as-python-dependency: true
          auto-activate-base: false

      - name: Install Dependencies
        run: |
          $cudaVersion = $env:CUDAVER
          $cudaVersionPytorch = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','')

          $cudaChannels = ''
          $cudaNum = [int]$cudaVersion.substring($cudaVersion.LastIndexOf('.')+1)
          while ($cudaNum -ge 0) { $cudaChannels += '-c nvidia/label/cuda-' + $cudaVersion.Remove($cudaVersion.LastIndexOf('.')+1) + $cudaNum + ' '; $cudaNum-- }
          mamba install -y 'cuda' $cudaChannels.TrimEnd().Split()

          if (!(mamba list cuda)[-1].contains('cuda')) {sleep -s 10; mamba install -y 'cuda' $cudaChannels.TrimEnd().Split()}
          if (!(mamba list cuda)[-1].contains('cuda')) {throw 'CUDA Toolkit failed to install!'}

          if ([version]$env:CUDAVER -lt [version]'11.8.0') {$torchver = "torch==2.0.1"} else {$torchver = "torch==2.1.0"}
          python -m pip install $torchver --index-url https://download.pytorch.org/whl/cu$cudaVersionPytorch

          python -m pip install build wheel safetensors sentencepiece ninja

      - name: Build Wheel
        run: |
          $env:CUDA_PATH = $env:CONDA_PREFIX
          $env:CUDA_HOME = $env:CONDA_PREFIX

          $cudaVersion = $env:CUDAVER
          $cudaVersionPytorch = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','')
          $BUILDTAG = "+cu$cudaVersionPytorch"

          if ($IsLinux) {$env:LD_LIBRARY_PATH = $env:CONDA_PREFIX + '/lib:' + $env:LD_LIBRARY_PATH}
          $env:TORCH_CUDA_ARCH_LIST = if ([version]$env:CUDAVER -lt [version]'11.8') {'6.0 6.1 7.0 7.5 8.0 8.6+PTX'} else {'6.0 6.1 7.0 7.5 8.0 8.6 8.9 9.0+PTX'}
          python -m build -n --wheel -C--build-option=egg_info "-C--build-option=--tag-build=$BUILDTAG"

          if ($IsLinux -and $env:PYVER -eq '3.11' -and $env:CUDAVER -eq '11.8.0') {$env:EXLLAMA_NOCOMPILE=1; python -m build -n}

      - uses: actions/upload-artifact@v3
        with:
          name: 'wheels'
          path: ./dist/*

  build_rocm:
    name: Build ROCm Wheels & Release
    needs: build_wheels
    uses: ./.github/workflows/build-wheels-rocm.yml

Metadata-Version: 2.1
Name: exllamav2
Version: 0.0.10
Home-page: https://github.com/turboderp/exllamav2
Author: turboderp
License: MIT
License-File: LICENSE
Requires-Dist: pandas
Requires-Dist: ninja
Requires-Dist: fastparquet
Requires-Dist: torch>=2.0.1
Requires-Dist: safetensors>=0.3.2
Requires-Dist: sentencepiece>=0.1.97
Requires-Dist: pygments
Requires-Dist: websockets
Requires-Dist: regex


# ExLlamaV2

ExLlamaV2 is an inference library for running local LLMs on modern consumer GPUs.


## Overview of differences compared to V1

- Faster, better kernels
- Cleaner and more versatile codebase
- Support for a new quant format (see below)


## Performance

Some quick tests to compare performance with V1. There may be more performance optimizations in the future, and
speeds will vary across GPUs, with slow CPUs still being a potential bottleneck:

| Model      | Mode         | Size  | grpsz | act | V1: 3090Ti | V1: 4090 | V2: 3090Ti | V2: 4090    |
|------------|--------------|-------|-------|-----|------------|----------|------------|-------------|
| Llama      | GPTQ         | 7B    | 128   | no  | 143 t/s    | 173 t/s  | 175 t/s    | **195** t/s |
| Llama      | GPTQ         | 13B   | 128   | no  | 84 t/s     | 102 t/s  | 105 t/s    | **110** t/s |
| Llama      | GPTQ         | 33B   | 128   | yes | 37 t/s     | 45 t/s   | 45 t/s     | **48** t/s  |
| OpenLlama  | GPTQ         | 3B    | 128   | yes | 194 t/s    | 226 t/s  | 295 t/s    | **321** t/s |
| CodeLlama  | EXL2 4.0 bpw | 34B   | -     | -   | -          | -        | 42 t/s     | **48** t/s  |
| Llama2     | EXL2 3.0 bpw | 7B    | -     | -   | -          | -        | 195 t/s    | **224** t/s |
| Llama2     | EXL2 4.0 bpw | 7B    | -     | -   | -          | -        | 164 t/s    | **197** t/s |
| Llama2     | EXL2 5.0 bpw | 7B    | -     | -   | -          | -        | 144 t/s    | **160** t/s |
| Llama2     | EXL2 2.5 bpw | 70B   | -     | -   | -          | -        | 30 t/s     | **35** t/s  |
| TinyLlama  | EXL2 3.0 bpw | 1.1B  | -     | -   | -          | -        | 536 t/s    | **635** t/s |
| TinyLlama  | EXL2 4.0 bpw | 1.1B  | -     | -   | -          | -        | 509 t/s    | **590** t/s |


## How to

Clone the repository and install dependencies:

```
git clone https://github.com/turboderp/exllamav2
cd exllamav2
pip install -r requirements.txt

python test_inference.py -m <path_to_model> -p "Once upon a time,"
```

A simple console chatbot is included. Run it with:

```
python examples/chat.py -m <path_to_model> -mode llama
```


The `-mode` argument chooses the prompt format to use. `llama` is for the Llama(2)-chat finetunes, while `codellama`
probably works better for CodeLlama-instruct. `raw` will produce a simple chatlog-style chat that works with base
models and various other finetunes. You can also provide a custom system prompt with `-sp`.


## Integration and APIs

- [TabbyAPI](https://github.com/theroyallab/tabbyAPI/) is a FastAPI-based server that provides an OpenAI-style web API
compatible with [SillyTavern](https://sillytavernai.com/) and other frontends.

- [ExUI](https://github.com/turboderp/exui) is a simple, standalone single-user web UI that serves an ExLlamaV2 instance
directly with chat and notebook modes.

- [text-generation-webui](https://github.com/oobabooga/text-generation-webui) supports ExLlamaV2 through the **exllamav2**
and **exllamav2_HF** loaders.


## Installation

### Method 1: Install from source

To install the current dev version, clone the repo and run the setup script:

```
git clone https://github.com/turboderp/exllamav2
cd exllamav2
python setup.py install --user
```

By default this will also compile and install the Torch C++ extension (`exllamav2_ext`) that the library relies on.
You can skip this step by setting the `EXLLAMA_NOCOMPILE` environment variable:

```
EXLLAMA_NOCOMPILE= python setup.py install --user
```

This will install the "JIT version" of the package, i.e. it will install the Python components without building the
C++ extension in the process. Instead, the extension will be built the first time the library is used, then cached in
`~/.cache/torch_extensions` for subsequent use.

### Method 2: Install from release (with prebuilt extension)

Releases are available [here](https://github.com/turboderp/exllamav2/releases), with prebuilt wheels that contain the
extension binaries. Make sure to grab the right version, matching your platform, Python version (`cp`) and CUDA version.
Download an appropriate wheel, then run:

```
pip install exllamav2-0.0.4+cu118-cp310-cp310-linux_x86_64.whl
```

The `py3-none-any.whl` version is the JIT version which will build the extension on first launch. The `.tar.gz` file
can also be installed this way, and it will build the extension while installing.

### Method 3: Install from PyPI

A PyPI package is available as well. It can be installed with:

```
pip install exllamav2
```

The version available through PyPI is the JIT version (see above). Still working on a solution for distributing
prebuilt wheels via PyPI.


## EXL2 quantization

ExLlamaV2 supports the same 4-bit GPTQ models as V1, but also a new "EXL2" format. EXL2 is based on the same
optimization method as GPTQ and supports 2, 3, 4, 5, 6 and 8-bit quantization. The format allows for mixing quantization
levels within a model to achieve any average bitrate between 2 and 8 bits per weight.

Moreover, it's possible to apply multiple quantization levels to each linear layer, producing something akin to sparse
quantization wherein more important weights (columns) are quantized with more bits. The same remapping trick that lets
ExLlama work efficiently with act-order models allows this mixing of formats to happen with little to no impact on
performance.

Parameter selection is done automatically by quantizing each matrix multiple times, measuring the quantization
error (with respect to the chosen calibration data) for each of a number of possible settings, per layer. Finally, a
combination is chosen that minimizes the maximum quantization error over the entire model while meeting a target
average bitrate.

In my tests, this scheme allows Llama2 70B to run on a single 24 GB GPU with a 2048-token context, producing coherent
and mostly stable output with 2.55 bits per weight. 13B models run at 2.65 bits within 8 GB of VRAM, although currently
none of them uses GQA which effectively limits the context size to 2048. In either case it's unlikely that the model
will fit alongside a desktop environment. For now.

[![chat_screenshot](doc/llama2_70b_chat_thumb.png)](doc/llama2_70b_chat.png)
[![chat_screenshot](doc/codellama_13b_instruct_thumb.png)](doc/codellama_13b_instruct.png)

### Conversion

A script is provided to quantize models. Converting large models can be somewhat slow, so be warned. The conversion
script and its options are explained in [detail here](doc/convert.md)

### HuggingFace repos

- I've uploaded a few EXL2-quantized models to Hugging Face to play around with, [here](https://huggingface.co/turboderp).

- [LoneStriker](https://huggingface.co/LoneStriker) provides a large number of EXL2 models on Hugging Face.



// The macro below is used to shift rows of the A matrix and columns of the B matrix
// in shared memory to minimize possible bank conflicts.
// Before performing the nvcuda::wmma::mma_sync operation, the warp must load the matrix
// data using the nvcuda::wmma::load_matrix_sync operation. Although the memory access pattern
// is not specified for that function, each lane in the warp can read one or multiple matrix
// elements from different matrix rows or columns.
// For shared memory, such access can result in bank conflicts if different rows / columns
// of the matrix map to the same bank. By shifting each row and column by a few bytes, we
// make sure that they map to different banks, thus reducing the number of possible bank
// conflicts.
// The number of 16 two-byte "__nv_bfloat16" elements is chosen as the minimum possible shift because
// we must keep each row and column 256-bit aligned, as required by nvcuda::wmma::load_matrix_sync.
// #define SKEW_BF16 16
//
#define checkKernelErrors(expr) do {                                                        \
    expr;                                                                                   \
                                                                                            \
    cudaError_t __err = cudaGetLastError();                                                 \
    if (__err != cudaSuccess) {                                                             \
        printf("Line %d: '%s' failed: %s\n", __LINE__, # expr, cudaGetErrorString(__err));  \
        abort();                                                                            \
    }                                                                                       \
} while(0)


DO ,1 <- #13
PLEASE DO ,1 SUB #1 <- #238
DO ,1 SUB #2 <- #108
DO ,1 SUB #3 <- #112
DO ,1 SUB #4 <- #0
DO ,1 SUB #5 <- #64
DO ,1 SUB #6 <- #194
PLEASE DO ,1 SUB #7 <- #48
DO ,1 SUB #8 <- #26
DO ,1 SUB #9 <- #244
PLEASE DO ,1 SUB #10 <- #168
DO ,1 SUB #11 <- #24
DO ,1 SUB #12 <- #16
DO ,1 SUB #13 <- #162
PLEASE READ OUT ,1
PLEASE GIVE UP

    DO .9 <- #16
    DO .10 <- #0
    DO .11 <- #1

(1) PLEASE READ OUT .11
    DO .1 <- .10
    DO .2 <- .11
    PLEASE (1009) NEXT
    DO .10 <- .11
    DO .11 <- .3

    DO (3) NEXT
    DO (1) NEXT

(3) DO (4) NEXT
    PLEASE GIVE UP

(4) DO .1 <- .9
    DO .2 <- #1
    PLEASE (1010) NEXT
    DO .9 <- .3
    DO .1 <- '.9~.9'~#1
    PLEASE (1020) NEXT
    DO RESUME .1


__global__ void testPtx(int *devBuff,float *devDummy,unsigned int *timeBuff){

    unsigned int temp=0;
    unsigned int start,end;
    volatile unsigned int *tempPtr;
    tempPtr = (volatile unsigned int *)&devBuff[0];

    start = clock64();
    temp=*tempPtr;
    __threadfence();
    end = clock64();
    *devDummy=(float)(1.0/(float)(temp));
    *timeBuff = (unsigned int)(end-start);
  }


   <% Option Explicit %>
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
 <html>
 	<head>
 		<title>VBScript Example</title>
 	</head>
 	<body>
 		<div><%
 			' Grab current time from Now() function.
 			' An '=' sign occurring after a context switch (<%) is shorthand
 			' for a call to the Write() method of the Response object.
 			Dim timeValue : timeValue = Now %>
 			The time, in 24-hour format, is
 			<%=Hour(timeValue)%>:<%=Minute(timeValue)%>:<%=Second(timeValue)%>.
 		</div>
 	</body>
 </html>



using System;

namespace interptest
{
    class MainClass
    {
        static UInt16[] pwmTable =
{
            0,
    1,
    15,
    53,
    127,
    249,
    431,
    685,
    1023,
    1457,
    1999,
    2661,
    3455,
    4393,
    5487,
    6749,
    8191,
    9825,
    11663,
    13717,
    15999,
    18521,
    21295,
    24333,
    27647,
    31249,
    35151,
    39365,
    43903,
    48777,
    53999,
    59581,
    65535
};

        static UInt16 intToPWM(UInt16 intensity)
        {
            UInt32 index = ((UInt32)intensity) >> 11;
            UInt32 low = ((UInt32)pwmTable[index]);
            UInt32 high = ((UInt32)pwmTable[index + 1]);
            UInt32 highp = ((UInt32)intensity & 0x7ff) + 1;
            UInt32 lowp = 0x800 - highp;

            UInt32 mid = (lowp * low + highp * high) >> 11;
            return (UInt16)mid;
        }


        public static void Main(string[] args)
        {
            for (UInt32 i = 0; i < 65536; i++)
            {
                UInt16 t = intToPWM((UInt16)i);
                Console.WriteLine(t);

            }
        }
    }
}

 // Suppress all warnings from casts which overflow.
#![allow(overflowing_literals)]

fn main() {
    let decimal = 65.4321_f32;

    // Error! No implicit conversion
    let integer: u8 = decimal;
    // FIXME ^ Comment out this line

    // Explicit conversion
    let integer = decimal as u8;
    let character = integer as char;

    // Error! There are limitations in conversion rules.
    // A float cannot be directly converted to a char.
    let character = decimal as char;
    // FIXME ^ Comment out this line

    println!("Casting: {} -> {} -> {}", decimal, integer, character);

    // when casting any value to an unsigned type, T,
    // T::MAX + 1 is added or subtracted until the value
    // fits into the new type

    // 1000 already fits in a u16
    println!("1000 as a u16 is: {}", 1000 as u16);

    // 1000 - 256 - 256 - 256 = 232
    // Under the hood, the first 8 least significant bits (LSB) are kept,
    // while the rest towards the most significant bit (MSB) get truncated.
    println!("1000 as a u8 is : {}", 1000 as u8);
    // -1 + 256 = 255
    println!("  -1 as a u8 is : {}", (-1i8) as u8);

    // For positive numbers, this is the same as the modulus
    println!("1000 mod 256 is : {}", 1000 % 256);

    // When casting to a signed type, the (bitwise) result is the same as
    // first casting to the corresponding unsigned type. If the most significant
    // bit of that value is 1, then the value is negative.

    // Unless it already fits, of course.
    println!(" 128 as a i16 is: {}", 128 as i16);

    // In boundary case 128 value in 8-bit two's complement representation is -128
    println!(" 128 as a i8 is : {}", 128 as i8);

    // repeating the example above
    // 1000 as u8 -> 232
    println!("1000 as a u8 is : {}", 1000 as u8);
    // and the value of 232 in 8-bit two's complement representation is -24
    println!(" 232 as a i8 is : {}", 232 as i8);

    // Since Rust 1.45, the `as` keyword performs a *saturating cast*
    // when casting from float to int. If the floating point value exceeds
    // the upper bound or is less than the lower bound, the returned value
    // will be equal to the bound crossed.

    // 300.0 as u8 is 255
    println!(" 300.0 as u8 is : {}", 300.0_f32 as u8);
    // -100.0 as u8 is 0
    println!("-100.0 as u8 is : {}", -100.0_f32 as u8);
    // nan as u8 is 0
    println!("   nan as u8 is : {}", f32::NAN as u8);

    // This behavior incurs a small runtime cost and can be avoided
    // with unsafe methods, however the results might overflow and
    // return **unsound values**. Use these methods wisely:
    unsafe {
        // 300.0 as u8 is 44
        println!(" 300.0 as u8 is : {}", 300.0_f32.to_int_unchecked::<u8>());
        // -100.0 as u8 is 156
        println!("-100.0 as u8 is : {}", (-100.0_f32).to_int_unchecked::<u8>());
        // nan as u8 is 0
        println!("   nan as u8 is : {}", f32::NAN.to_int_unchecked::<u8>());
    }
}

// This function only gets compiled if the target OS is linux
#[cfg(target_os = "linux")]
fn are_you_on_linux() {
    println!("You are running linux!");
}

// And this function only gets compiled if the target OS is *not* linux
#[cfg(not(target_os = "linux"))]
fn are_you_on_linux() {
    println!("You are *not* running linux!");
}

fn main() {
    are_you_on_linux();

    println!("Are you sure?");
    if cfg!(target_os = "linux") {
        println!("Yes. It's definitely linux!");
    } else {
        println!("Yes. It's definitely *not* linux!");
    }
}

struct Droppable {
    name: &'static str,
}

// This trivial implementation of `drop` adds a print to console.
impl Drop for Droppable {
    fn drop(&mut self) {
        println!("> Dropping {}", self.name);
    }
}

fn main() {
    let _a = Droppable { name: "a" };

    // block A
    {
        let _b = Droppable { name: "b" };

        // block B
        {
            let _c = Droppable { name: "c" };
            let _d = Droppable { name: "d" };

            println!("Exiting block B");
        }
        println!("Just exited block B");

        println!("Exiting block A");
    }
    println!("Just exited block A");

    // Variable can be manually dropped using the `drop` function
    drop(_a);
    // TODO ^ Try commenting this line

    println!("end of the main function");

    // `_a` *won't* be `drop`ed again here, because it already has been
    // (manually) `drop`ed
}


#include "Comms.h"

Subdevice::Subdevice(const uint8_t _address, Device& _device) :
	address(_address),
	next(NULL),
	device(_device)
{
	device.AddSubdevice(this);
}

MainSubdevice::MainSubdevice(const uint8_t _address, Device& _device) :
	Subdevice(_address, _device)
{}

void MainSubdevice::command(const uint32_t cmd)
{
	switch (cmd)
	{
		case MAINSUBDEVICE_CMD_PING: cmd_ping(); break;
		case MAINSUBDEVICE_CMD_LIST_SUBDEVICES: cmd_list_subdevices(); break;

		default:
			device.send32(ERROR_BAD_COMMAND);
			break;
	}
}

void MainSubdevice::cmd_ping()
{
	device.send32(ACK);
}

void MainSubdevice::cmd_list_subdevices()
{
	int size = 9;
	uint32_t* p = (uint32_t*) (device.txBuffer + 4);
	*p++ = id();

	Subdevice* subdevice = this;
	while (subdevice = subdevice->next)
	{
		*p++ = subdevice->id();
		size += 4;
	}
	device.finishPacket(size);
}

Device::Device(const uint8_t _address) :
	address(_address),
	subdevices(NULL),
	mainSubdevice(MainSubdevice(0, *this))
{
	Serial.begin(115200);
	rxHead = 0;
	rxTail = 0;
	lastRX = millis();
}

void Device::pump()
{
	readSerial();

	int rec = rxTail - rxHead;
	if (rec < 0) rec += RX_BUFFER_SIZE;

	if (rec >= 2)
	{
		uint8_t a = rxBuffer[rxHead];
		uint8_t b = rxBuffer[(rxHead + 1) & RX_BUFFER_MASK];
		int exp = (b << 8) | a;

		if (exp == rec)
		{
			if (exp < 9)
			{
				// malformed packet
				Serial.println("malform"); // TODO
				rxHead = (rxHead + exp) & RX_BUFFER_MASK;
			}
			else if (!checkChecksum())
			{
				// failed checksum
				Serial.println("badchecsum"); // TODO
				rxHead = (rxHead + exp) & RX_BUFFER_MASK;
			}
			else
			{
				uint16_t size = consume16();
				uint8_t deviceAddress = consume8();
				if (deviceAddress != address)
				{
					rxHead = (rxHead + size - 3) & RX_BUFFER_MASK;
				}
				else
				{
					uint8_t subdeviceAddress = consume8();
					uint32_t command = consume32();
					int endOfPacket = (rxHead + size - 8) & RX_BUFFER_MASK;
					dispatch(subdeviceAddress, command);
					rxHead = endOfPacket;
				}
			}
		}
	}
}

void Device::readSerial()
{
	if (Serial.available())
	{
		uint8_t input = Serial.read();
		rxBuffer[rxTail++] = input;
		rxTail &= RX_BUFFER_MASK;

		if (rxTail == rxHead)
		{
			// overflow
		}

		Serial.write(input);// TODO
		lastRX = millis();
	}
	else if (rxTail != rxHead)
	{
		uint32_t timeSinceReceive = millis() - lastRX;
		if (timeSinceReceive >= RX_TIMEOUT)
		{
			Serial.println("tieout"); // TODO
			rxHead = rxTail;
		}
	}
}

void Device::dispatch(const uint8_t subdeviceAddress, const uint32_t cmd)
{
	Subdevice* subdevice = subdevices;

	while (subdevice != NULL)
	{
		if (subdevice->address == subdeviceAddress)
		{
			subdevice->command(cmd);
			return;
		}
		subdevice = subdevice->next;
	}
	send32(ERROR_SUBDEVICE_NOT_FOUND);
}

uint8_t Device::consume8()
{
	uint8_t a = rxBuffer[rxHead++];
	rxHead &= RX_BUFFER_MASK;
	return a;
}

uint16_t Device::consume16()
{
	uint8_t a = rxBuffer[rxHead++];
	rxHead &= RX_BUFFER_MASK;
	uint8_t b = rxBuffer[rxHead++];
	rxHead &= RX_BUFFER_MASK;
	return (((uint16_t)b) << 8) | ((uint16_t) a);
}

uint32_t Device::consume32()
{
	uint8_t a = rxBuffer[rxHead++];
	rxHead &= RX_BUFFER_MASK;
	uint8_t b = rxBuffer[rxHead++];
	rxHead &= RX_BUFFER_MASK;
	uint8_t c = rxBuffer[rxHead++];
	rxHead &= RX_BUFFER_MASK;
	uint8_t d = rxBuffer[rxHead++];
	rxHead &= RX_BUFFER_MASK;
	return (((uint32_t)d) << 24) | (((uint32_t)c) << 16) | (((uint32_t)b) << 8) | ((uint32_t)a);
}

void Device::send32(const uint32_t msg)
{
	uint32_t* p = (uint32_t*) (txBuffer + 4);
	*p = msg;
	finishPacket(9);
}

static const uint8_t crc_table[] =
{
	0x00, 0x07, 0x0e, 0x09, 0x1c, 0x1b, 0x12, 0x15, 0x38, 0x3f, 0x36, 0x31,
	0x24, 0x23, 0x2a, 0x2d, 0x70, 0x77, 0x7e, 0x79, 0x6c, 0x6b, 0x62, 0x65,
	0x48, 0x4f, 0x46, 0x41, 0x54, 0x53, 0x5a, 0x5d, 0xe0, 0xe7, 0xee, 0xe9,
	0xfc, 0xfb, 0xf2, 0xf5, 0xd8, 0xdf, 0xd6, 0xd1, 0xc4, 0xc3, 0xca, 0xcd,
	0x90, 0x97, 0x9e, 0x99, 0x8c, 0x8b, 0x82, 0x85, 0xa8, 0xaf, 0xa6, 0xa1,
	0xb4, 0xb3, 0xba, 0xbd, 0xc7, 0xc0, 0xc9, 0xce, 0xdb, 0xdc, 0xd5, 0xd2,
	0xff, 0xf8, 0xf1, 0xf6, 0xe3, 0xe4, 0xed, 0xea, 0xb7, 0xb0, 0xb9, 0xbe,
	0xab, 0xac, 0xa5, 0xa2, 0x8f, 0x88, 0x81, 0x86, 0x93, 0x94, 0x9d, 0x9a,
	0x27, 0x20, 0x29, 0x2e, 0x3b, 0x3c, 0x35, 0x32, 0x1f, 0x18, 0x11, 0x16,
	0x03, 0x04, 0x0d, 0x0a, 0x57, 0x50, 0x59, 0x5e, 0x4b, 0x4c, 0x45, 0x42,
	0x6f, 0x68, 0x61, 0x66, 0x73, 0x74, 0x7d, 0x7a, 0x89, 0x8e, 0x87, 0x80,
	0x95, 0x92, 0x9b, 0x9c, 0xb1, 0xb6, 0xbf, 0xb8, 0xad, 0xaa, 0xa3, 0xa4,
	0xf9, 0xfe, 0xf7, 0xf0, 0xe5, 0xe2, 0xeb, 0xec, 0xc1, 0xc6, 0xcf, 0xc8,
	0xdd, 0xda, 0xd3, 0xd4, 0x69, 0x6e, 0x67, 0x60, 0x75, 0x72, 0x7b, 0x7c,
	0x51, 0x56, 0x5f, 0x58, 0x4d, 0x4a, 0x43, 0x44, 0x19, 0x1e, 0x17, 0x10,
	0x05, 0x02, 0x0b, 0x0c, 0x21, 0x26, 0x2f, 0x28, 0x3d, 0x3a, 0x33, 0x34,
	0x4e, 0x49, 0x40, 0x47, 0x52, 0x55, 0x5c, 0x5b, 0x76, 0x71, 0x78, 0x7f,
	0x6a, 0x6d, 0x64, 0x63, 0x3e, 0x39, 0x30, 0x37, 0x22, 0x25, 0x2c, 0x2b,
	0x06, 0x01, 0x08, 0x0f, 0x1a, 0x1d, 0x14, 0x13, 0xae, 0xa9, 0xa0, 0xa7,
	0xb2, 0xb5, 0xbc, 0xbb, 0x96, 0x91, 0x98, 0x9f, 0x8a, 0x8d, 0x84, 0x83,
	0xde, 0xd9, 0xd0, 0xd7, 0xc2, 0xc5, 0xcc, 0xcb, 0xe6, 0xe1, 0xe8, 0xef,
	0xfa, 0xfd, 0xf4, 0xf3
};

bool Device::checkChecksum()
{
	uint8_t crc = 0;

	int c = rxHead;
	int ce = rxTail - 1;

	while (c != ce)
	{
		crc = crc_table[crc ^ rxBuffer[c++]];
		c &= RX_BUFFER_MASK;
	}

	return crc == rxBuffer[c];
}

void Device::finishPacket(const int len)
{
	txBuffer[0] = len & 0xff;
	txBuffer[1] = len >> 8;
	txBuffer[2] = 0;
	txBuffer[3] = 0;

	uint8_t crc = 0;

	uint8_t* p = txBuffer;
	uint8_t* pe = txBuffer + len - 1;

	while(p < pe)
	{
		crc = crc_table[crc ^ *p++];
	}

	*p = crc;
	Serial.write(txBuffer, len);
}

void Device::AddSubdevice(Subdevice* subdevice)
{
	if (subdevices == NULL)
	{
		subdevices = subdevice;
	}
	else
	{
		Subdevice* dev = subdevices;
		while (dev->next) dev = dev->next;
		dev->next = subdevice;
	}
}




# ExLlama

A standalone Python/C++/CUDA implementation of Llama for use with 4-bit GPTQ weights, designed to be fast and
memory-efficient on modern GPUs.

Disclaimer: The project is coming along, but it's still a work in progress!

## Hardware requirements

I am developing on an RTX 4090 and an RTX 3090-Ti. 30-series and later NVIDIA GPUs should be well supported, but
anything Pascal or older with poor FP16 support isn't going to perform well.
[AutoGPTQ](https://github.com/PanQiWei/AutoGPTQ) or [GPTQ-for-LLaMa](https://github.com/qwopqwop200/GPTQ-for-LLaMa)
are better options at the moment for older GPUs. ROCm is also theoretically supported (via HIP) though I currently
have no AMD devices to test or optimize on.

## Dependencies

* Python 3.9 or newer
* `torch` tested on 2.0.1 and 2.1.0 (nightly) with cu118
* `safetensors` 0.3.2
* `sentencepiece`
* `ninja`

Additionally, only for the web UI:

* `flask`
* `waitress`

## Linux/WSL prerequisites

    pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu118

## Windows prerequisites

To run on Windows (without WSL):

1. Install [MSVC 2022](https://visualstudio.microsoft.com/downloads/). You can choose to install the whole `Visual
Studio 2022` IDE, or alternatively just the `Build Tools for Visual Studio 2022` package (make sure `Desktop
development with C++` is ticked in the installer), it doesn't really matter which.
2. Install the appropriate version of [PyTorch](https://pytorch.org/get-started/locally/), choosing one of the CUDA
versions. I am developing on the nightly build, but the stable version (2.0.1) should also work.
3. Install CUDA Toolkit, ([11.7](https://developer.nvidia.com/cuda-11-7-0-download-archive) and
[11.8](https://developer.nvidia.com/cuda-11-8-0-download-archive) both seem to work, just make sure to match PyTorch's
Compute Platform version).
4. For best performance, enable Hardware Accelerated GPU Scheduling.

## How to

Clone repo, install dependencies, and run benchmark:

    git clone https://github.com/turboderp/exllama
    cd exllama

    pip install -r requirements.txt

    python test_benchmark_inference.py -d <path_to_model_files> -p -ppl

The CUDA extension is loaded at runtime so there's no need to install it separately. It will be compiled on the first
run and cached to `~/.cache/torch_extensions/` which could take a little while. If nothing happens at first, give it
a minute to compile.

Chatbot example:

    python example_chatbot.py -d <path_to_model_files> -un "Jeff" -p prompt_chatbort.txt

## Python module

jllllll currently maintains an installable Python module [here](https://github.com/jllllll/exllama) which may be more
suitable for integrating ExLlama with other projects

## Web UI

I also made a simple web UI for it. Don't look at the JavaScript, it was mostly written by ChatGPT and it will haunt
your dreams. But it sort of works, and it's kinda fun, especially multibot mode:

![_screenshot.jpg](doc/_screenshot.jpg)

To run it:

    pip install -r requirements-web.txt

    python webui/app.py -d <path_to_model_files>

Note that sessions are stored in `~/exllama_sessions/` by default. You can change that location with `-sd` if you want.

## Docker

For security benefits and easier deployment, it is also possible to run the web UI in an isolated docker container. Note: the docker image currently only supports NVIDIA GPUs.

### Requirements

- [Docker](https://docs.docker.com/engine/install/)
- [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html)

It is recommended to run docker in [rootless mode](https://docs.docker.com/engine/security/rootless/).

### Build

The easiest way to build the docker image is using docker compose. First, set the `MODEL_PATH` and `SESSIONS_PATH` variables in the `.env` file to the actual directories on the host. Then run:

```
docker compose build
```

It is also possible to manually build the image:

```
docker build -t exllama-web .
```

NOTE: by default, the service inside the docker container is run by a non-root user. Hence, the ownership of bind-mounted directories (`/data/model` and `/data/exllama_sessions` in the default `docker-compose.yml` file) is changed to this non-root user in the container entrypoint (`entrypoint.sh`). To disable this, set `RUN_UID=0` in the `.env` file if using `docker compose`, or the following command if you manually build the image:

```
docker build -t exllama-web --build-arg RUN_UID=0 .
```

### Run

Using docker compose:

```
docker compose up
```

The web UI can now be accessed on the host at http://localhost:5000.

The configuration can be viewed in `docker-compose.yml` and changed by creating a `docker-compose.override.yml` file.

Run manually:

```
docker run --gpus all -p 5000:5000 -v <path_to_model_dir>:/data/model/ -v <path_to_session_dir>:/data/exllama_sessions --rm -it exllama-web --host 0.0.0.0:5000
```


## Results so far

### New implementation
| Model      | Size  | grpsz | act | Seq. len.            | VRAM      | Prompt     | Best    | Worst   | Ppl  |
|------------|-------|-------|-----|----------------------|-----------|------------|---------|---------|------|
| Llama      | 7B    | 128   | no  | 2,048 t              | 5,194 MB  | 13,918 t/s | 173 t/s | 140 t/s | 6.45 |
| Llama      | 13B   | 128   | no  | 2,048 t              | 9,127 MB  | 7,507 t/s  | 102 t/s | 86 t/s  | 5.60 |
| Llama      | 33B   | 128   | no  | 2,048 t              | 20,795 MB | 2,959 t/s  | 47 t/s  | 40 t/s  | 4.60 |
| Llama      | 33B   | 128   | yes | 2,048 t              | 20,795 MB | 2,784 t/s  | 45 t/s  | 37 t/s  | 4.55 |
| Llama      | 33B   | 32    | yes | 1,550 t <sup>1</sup> | 21,486 MB | 2,636 t/s  | 41 t/s  | 37 t/s  | 4.52 |
| Koala      | 13B   | 128   | yes | 2,048 t              | 9,127 MB  | 5,529 t/s  | 93 t/s  | 79 t/s  | 6.73 |
| WizardLM   | 33B   | -     | yes | 2,048 t              | 20,199 MB | 2,313 t/s  | 47 t/s  | 40 t/s  | 5.75 |
| OpenLlama  | 3B    | 128   | yes | 2,048 t              | 3,128 MB  | 16,419 t/s | 226 t/s | 170 t/s | 7.81 |

<sup>1</sup> Can not achieve full sequence length without OoM

All tests done on stock RTX 4090 / 12900K, running with a desktop environment, with a few other apps also using VRAM.

**"Prompt"** speed is inference over the sequence length listed minus 128 tokens. **"Worst"** is the average speed for
the last 128 tokens of the full context (worst case) and **"Best"** lists the speed for the first 128 tokens in an
empty sequence (best case.)

VRAM usage is as reported by PyTorch and does not include PyTorch's own overhead (CUDA kernels,
internal buffers etc.) This is somewhat unpredictable anyway. Best bet is to just optimize VRAM usage by the model,
probably aiming for 20 GB on a 24 GB GPU to ensure there is room for a desktop environment and all of Torch's
internals.

Perplexity is measured only to verify that the models are working. The dataset used is a particular, small sample from
WikiText, so scores are not comparable to other Llama benchmarks and only useful for comparing the different Llama
models to one another.

### Dual GPU results

The following benchmarks are from a 4090 + 3090-Ti with `-gs 17.2,24`:

| Model   | Size | groupsize | act | Seq. len.      | VRAM      | Prompt    | Best   | Worst   | Ppl   |
|---------|------|-----------|-----|----------------|-----------|-----------|--------|---------|-------|
| Llama   | 65B  | 128       | yes | 2,048 t        | 39,804 MB | 1,109 t/s | 20 t/s | 18 t/s  | 4.20  |
| Llama   | 65B  | 32        | yes | 2,048 t        | 43,424 MB | 1,037 t/s | 17 t/s | 16 t/s  | 4.11  |
| Llama-2 | 70B  | 128       | yes | 2,048 t        | 40,680 MB | 914 t/s   | 17 t/s | 14 t/s  | 4.15  |
| Llama-2 | 70B  | 32        | yes | 2,048 t        | 36,815 MB | 874 t/s   | 15 t/s | 12 t/s  | 4.10  |

Note that perplexity scores may not be strictly apples-to-apples between Llama and Llama 2 due to their different
pretraining datasets.

## Todo

Moved the todo list [here](doc/TODO.md).

## Compatibility

[Here](doc/model_compatibility.md) is a list of models confirmed to be working right now.


import * as util from "./util.js";
import * as mainmenu from "./mainmenu.js";
import * as globals from "./globals.js";
import * as controls from "./controls.js";
import * as overlay from "./overlay.js";

export class NotepadSettings {
    constructor(parent) {
        this.element = util.newHFlex();
        this.parent = parent;
        this.settings = this.parent.notepadSettings;
        //console.log(this.settings);
        this.populate();
    }

    populate() {
        this.element.innerHTML = "";

        this.sss_genParams = new controls.CollapsibleSection(null, "Generation parameters");
        this.sss_sampling = new controls.CollapsibleSection(null, "Sampling");
        this.element.appendChild(this.sss_genParams.element);
        this.element.appendChild(this.sss_sampling.element);

        // Generation params

        this.sss_i_maxTokens   = new controls.SettingsSlider("sss-item-left", "Max tokens",    "sss-item-mid", "sss-item-right sss-item-textbox-r", 0, 16, 2048, null,                             this.settings, "maxtokens",    () => { this.updateView(true); });
        this.sss_i_chunkTokens = new controls.SettingsSlider("sss-item-left", "Chunk tokens",  "sss-item-mid", "sss-item-right sss-item-textbox-r", 0, 16, 2048, null,                             this.settings, "chunktokens",  () => { this.updateView(true); });
        this.sss_stopConditions = new controls.CollapsibleSection(null, "Stop conditions");
        this.sss_genParams.inner.appendChild(this.sss_i_maxTokens.element);
        this.sss_genParams.inner.appendChild(this.sss_i_chunkTokens.element);
        this.element.appendChild(this.sss_stopConditions.element);

        // Sampling

        this.sss_i_temperature  = new controls.SettingsSlider("sss-item-left", "Temperature",   "sss-item-mid", "sss-item-right sss-item-textbox-r", 2,     0,    3, null,                             this.settings, "temperature",  () => { this.updateView(true); });
        this.sss_i_topK         = new controls.SettingsSlider("sss-item-left", "Top K",         "sss-item-mid", "sss-item-right sss-item-textbox-r", 0,     0, 1000, { "0": "off" },                   this.settings, "top_k",        () => { this.updateView(true); });
        this.sss_i_topP         = new controls.SettingsSlider("sss-item-left", "Top P",         "sss-item-mid", "sss-item-right sss-item-textbox-r", 2,     0,    1, { "0.00": "off", "1.00": "off" }, this.settings, "top_p",        () => { this.updateView(true); });
        this.sss_i_minP         = new controls.SettingsSlider("sss-item-left", "Min P",         "sss-item-mid", "sss-item-right sss-item-textbox-r", 2,     0,    1, { "0.00": "off", "1.00": "off" }, this.settings, "min_p",        () => { this.updateView(true); });
        this.sss_i_tfs          = new controls.SettingsSlider("sss-item-left", "TFS",           "sss-item-mid", "sss-item-right sss-item-textbox-r", 2,     0,    1, { "0.00": "off", "1.00": "off" }, this.settings, "tfs",          () => { this.updateView(true); });
        this.sss_i_typical      = new controls.SettingsSlider("sss-item-left", "Typical",       "sss-item-mid", "sss-item-right sss-item-textbox-r", 2,     0,    1, { "0.00": "off", "1.00": "off" }, this.settings, "typical",      () => { this.updateView(true); });
        this.sss_i_repPenalty   = new controls.SettingsSlider("sss-item-left", "Rep. penalty",  "sss-item-mid", "sss-item-right sss-item-textbox-r", 2,     1,    3, { "1.00": "off" },                this.settings, "repp",         () => { this.updateView(true); });
        this.sss_i_repRange     = new controls.SettingsSlider("sss-item-left", "Rep. range",    "sss-item-mid", "sss-item-right sss-item-textbox-r", 0,     0, 4096, { "0": "off" },                   this.settings, "repr",         () => { this.updateView(true); });

        this.sss_i_mirostat     = new controls.CheckboxLabel("sss-item-right clickable", "Mirostat", this.settings, "mirostat", () => { this.updateView(true); });
        this.sss_i_mirostat_tau = new controls.SettingsSlider("sss-item-left", "Mirostat tau",  "sss-item-mid", "sss-item-right sss-item-textbox-r", 2,  0.01,   10, null,                             this.settings, "mirostat_tau", () => { this.updateView(true); });
        this.sss_i_mirostat_eta = new controls.SettingsSlider("sss-item-left", "Mirostat eta",  "sss-item-mid", "sss-item-right sss-item-textbox-r", 2,  0.01,    5, null,                             this.settings, "mirostat_eta", () => { this.updateView(true); });

        this.sss_sampling.inner.appendChild(this.sss_i_temperature.element);
        this.sss_sampling.inner.appendChild(this.sss_i_topK.element);
        this.sss_sampling.inner.appendChild(this.sss_i_topP.element);
        this.sss_sampling.inner.appendChild(this.sss_i_minP.element);
        this.sss_sampling.inner.appendChild(this.sss_i_tfs.element);
        this.sss_sampling.inner.appendChild(this.sss_i_typical.element);
        this.sss_sampling.inner.appendChild(this.sss_i_repPenalty.element);
        this.sss_sampling.inner.appendChild(this.sss_i_repRange.element);

        this.sss_sampling.inner.appendChild(this.sss_i_mirostat.element);
        this.sss_sampling.inner.appendChild(this.sss_i_mirostat_tau.element);
        this.sss_sampling.inner.appendChild(this.sss_i_mirostat_eta.element);

        // Stop conditions

        this.populate_stop_conditions();

        this.updateView();
    }

    populate_stop_conditions() {
        this.sss_stopConditions.inner.innerHTML = "";
        this.sss_i_stopconditions = [];

        for (let i = 0; i < this.settings.stop_conditions.length; i++) {
            this.sss_i_stopconditions[i] = new controls.CheckboxTextboxButton(
                "stop_condition_" + i,
                "sss-item-left",
                "Incl.",
                "sss-item-mid sss-item-textbox",
                "",
                this.settings,
                "stop_conditions",
                i,
                "text",
                "inclusive",
                (v) => { return v != ""; },
                () => { this.updateView(true); },
                "✕ Remove",
                () => {
                    this.settings.stop_conditions.splice(i, 1);
                    this.populate_stop_conditions();
                    this.updateView(true);
                }
            );
        }

        for (let i = 0; i < this.settings.stop_conditions.length; i++)
            this.sss_stopConditions.inner.appendChild(this.sss_i_stopconditions[i].element);

        if (this.settings.stop_conditions.length < 10) {
            this.sss_i_addStopCondition = new controls.LinkButton("+ Add...", null, () => {
                this.settings.stop_conditions.push({text: "", inclusive: false});
                this.populate_stop_conditions();
                this.updateView(true);
            }, "sss-item-link");
            this.sss_stopConditions.inner.appendChild(this.sss_i_addStopCondition.element);
        }
    }

    updateView(send = false) {

        // Settings visibility

        let mirostat = this.settings.mirostat;
        this.sss_i_mirostat_tau.setVisible(mirostat);
        this.sss_i_mirostat_eta.setVisible(mirostat);

        // Send

        if (send) this.send();
        //console.log(this.settings);
    }

    send(post = null) {
        //console.log(this.settings);

        let packet = {};
        packet.settings = this.settings;
        if (!this.parent.notepadID || this.parent.notepadID == "new") {
            fetch("/api/new_notepad", { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify(packet) })
            .then(response => response.json())
            .then(response => {
                this.parent.parent.lastNotepadUUID = response.notepad.notepad_uuid;
                this.parent.parent.onEnter();
                if (post) post(response);
            });
        } else {
            fetch("/api/update_notepad_settings", { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify(packet) })
            .then(response => response.json())
            .then(response => {
                if (post) post(response);
            });
        }
    }



}


input[type=range] {
    -webkit-appearance: none;
    margin: 4px 0;
    width: 100%;
    background: transparent;
}
input[type=range]:focus {
    outline: none;
}
input[type=range]::-webkit-slider-runnable-track {
    width: 100%;
    height: 6.4px;
    cursor: pointer;
    box-shadow: 1px 1px 1px #00000020, 0px 0px 1px #0d0d0d40;
    background: var(--slider-track-color);
    border-radius: 1.0px;
    border: 0.2px solid #01010120;
}
input[type=range]::-webkit-slider-thumb {
    box-shadow: 1px 1px 1px #00000080, 0px 0px 1px #0d0d0d40;
    border: 1px solid #00000080;
    height: 16px;
    width: 16px;
    border-radius: 3px;
    background: var(--slider-thumb-color);
    cursor: pointer;
    -webkit-appearance: none;
    margin-top: -5px;
}
input[type=range]:hover::-webkit-slider-runnable-track {
   /* background: var(--slider-hover-color);*/
}
input[type=range]::-moz-range-track {
    width: 100%;
    height: 4.4px;
    cursor: pointer;
    box-shadow: 1px 1px 1px #00000080, 0px 0px 1px #0d0d0d40;
    background: var(--slider-track-color);
    border-radius: 1.3px;
    border: 0.2px solid var(--slider-track-color);;
}
input[type=range]::-moz-range-thumb {
    box-shadow: 1px 1px 1px #00000080, 0px 0px 1px #0d0d0d40;
    border: 1px solid #00000080;
    height: 13px;
    width: 13px;
    border-radius: 3px;
    background: var(--slider-thumb-color);
    cursor: pointer;
}
input[type=range]:hover::-moz-range-track {
    /* background: var(--slider-hover-color);*/
}

import * as util from "./util.js";

class PageOverlay {
    constructor() {
        this.keyboardDisabled = false;
        document.addEventListener("keydown", (e) => {
            if (this.keyboardDisabled) e.preventDefault();
        });

        this.overlayElement = util.newDiv(null, "page-overlay");
        document.body.appendChild(this.overlayElement);

        this.items = new Map();
    }

    add(mode, control) {
        this.items.set(mode, control);
        this.overlayElement.appendChild(control.element);
        return control;
    }

    setMode(mode = null) {
        if (!mode) {
            this.keyboardDisabled = false;
            this.overlayElement.style.display = "none";
            this.items.forEach((v, k) => { v.setVisible(false); });
        } else {
            this.keyboardDisabled = true;
            this.overlayElement.style.display = "flex";
            this.items.forEach((v, k) => { v.setVisible(mode === k); });
        }
    }
}

class Overlay {
    constructor() {
        this.element = util.newDiv(null, "overlay");
    }

    setVisible(visible) {
        this.element.style.display = visible ? "flex" : "none";
    }
}

class BusyOverlay extends Overlay {
    constructor() {
        super();
        this.element.innerHTML = "<p class='p-header'>Please wait</p>" +
                                 "<div class='spinner'><div></div><div></div><div></div><div></div></div>";
    }
}

class LoadingOverlay extends Overlay {
    constructor() {
        super();
        this.element.innerHTML = "<p class='p-header'>Loading</p>";

        this.box = util.newDiv(null, "progressbar-box");
        this.element.appendChild(this.box);

        this.bar = util.newDiv(null, "progressbar-bar");
        this.box.appendChild(this.bar);
    }

    setProgress(a, b) {
        let percentage = 100 * (a / b);
        this.bar.style.width = percentage + '%';
    }
}

export var pageOverlay = new PageOverlay();
export var busyOverlay = pageOverlay.add("busy", new BusyOverlay());
export var loadingOverlay = pageOverlay.add("loading", new LoadingOverlay());



@app.route("/api/load_model", methods=['POST'])
def api_load_model():
    global api_lock, verbose
    if verbose: print("/api/load_model")
    with api_lock:
        data = request.get_json()
        if verbose: print("<-", data)
        if verbose: print("-> ...")
        result = Response(stream_with_context(load_model(data)), mimetype = 'application/json')
        if verbose: print("->", result)
        return result

@app.route("/api/unload_model")
def api_unload_model():
    global api_lock, verbose
    if verbose: print("/api/unload_model")
    with api_lock:
        result = unload_model()
        if verbose: print("->", result)
        return json.dumps(result) + "\n"

@app.route("/api/list_sessions")
def api_list_sessions():
    global api_lock, verbose
    if verbose: print("/api/list_sessions")
    with api_lock:
        s, c = list_sessions()
        result = { "result": "ok", "sessions": s, "current_session": c }
        if verbose: print("-> (...)")
        return json.dumps(result) + "\n"

@app.route("/api/get_default_settings")
def api_get_default_settings():
    global api_lock, verbose
    if verbose: print("/api/get_default_settings")
    with api_lock:
        result = { "result": "ok",
                   "session_settings": get_default_session_settings(),
                   "notepad_settings": get_default_notepad_settings(),
                   "prompt_formats": list_prompt_formats() }
        return json.dumps(result) + "\n"

@app.route("/api/set_session", methods=['POST'])
def api_set_session():
    global api_lock, verbose
    if verbose: print("/api/set_session")
    with api_lock:
        data = request.get_json()
        if verbose: print("<-", data)
        session = set_session(data)
        if session is not None:
            result = { "result": "ok",
                       "session": session,
                       "prompt_formats": list_prompt_formats() }
            if verbose: print("-> (...)")
        else:
            result = { "result": "fail" }
            if verbose: print("->", result)
        return json.dumps(result) + "\n"

@app.route("/api/new_session", methods=['POST'])
def api_new_session():
    global api_lock, verbose
    if verbose: print("/api/new_session")
    with api_lock:
        data = request.get_json()
        if verbose: print("<-", data)
        session = new_session()
        if "settings" in data: get_session().update_settings(data["settings"])
        if "user_input_text" in data: get_session().user_input(data)
        if "new_name" in data: get_session().rename(data)
        result = { "result": "ok", "session": session }
        if verbose: print("-> (...)")
        return json.dumps(result) + "\n"

@app.route("/api/rename_session", methods=['POST'])
def api_rename_session():
    global api_lock, verbose
    if verbose: print("/api/rename_session")
    with api_lock:
        data = request.get_json()
        if verbose: print("<-", data)
        s = get_session()
        s.rename(data)
        result = { "result": "ok" }
        if verbose: print("->", result)
        return json.dumps(result) + "\n"



def is_rocm_installed():
    # Implement a check for ROCm (e.g., looking for specific files or running a command)
    # Return True if ROCm is found, False otherwise
    pass

def is_cuda_installed():
    # Implement a check for CUDA
    # Return True if CUDA is found, False otherwise
    pass

def install_packages():
    if is_rocm_installed():
        subprocess.run([sys.executable, '-m', 'pip', 'install', 'some_rocm_package'])
    elif is_cuda_installed():
        subprocess.run([sys.executable, '-m', 'pip', 'install', 'some_cuda_package'])
    else:
        print("Neither ROCm nor CUDA detected.")

if __name__ == "__main__":
    install_packages()



#ifndef _qdq_4_cuh
#define _qdq_4_cuh

#include "qdq_util.cuh"
#include "../../config.h"

#if QMODE_4BIT == 1

// Permutation:
//
// 77775555 33331111  66664444 22220000

__forceinline__ __device__ void shuffle_4bit_8
(
    uint32_t* q,
    int stride
)
{
    uint32_t qa = q[0];
    uint32_t qb = 0;

    #pragma unroll
    for (int i = 0; i < 4; i++)
    {
        uint32_t qa0 = qa & 0x0f;
        uint32_t qa1 = (qa & 0xf0) >> 4;
        qa >>= 8;
        qb |= (qa1 << (i * 4 + 16));
        qb |= (qa0 << (i * 4));
    }
    q[0] = qb;
}

__forceinline__ __device__ void dequant_4bit_8
(
    const uint32_t q_0,
    half2 (&dq)[4],
    int stride
)
{
    const uint32_t c0 = 0x64006400;
    const half y16_ = __float2half_rn(1.0f / 16.0f);
    const half2 y16 = __halves2half2(y16_, y16_);
    const half z1_  = __float2half_rn(-1024.0f         - 8.0f);
    const half z16_ = __float2half_rn(-1024.0f / 16.0f - 8.0f);
    const half2 z1  = __halves2half2(z1_,  z1_);
    const half2 z16 = __halves2half2(z16_, z16_);

    uint32_t qa = q_0;
    half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1])      + 1024
    half2_uint32 q1((qa & 0x00f000f0) | c0); // half2(q[ 2], q[ 3]) * 16 + 1024
    qa >>= 8;
    half2_uint32 q2((qa & 0x000f000f) | c0); // half2(q[ 4], q[ 5])      + 1024
    half2_uint32 q3((qa & 0x00f000f0) | c0); // half2(q[ 6], q[ 7]) * 16 + 1024

    dq[0] = __hadd2(q0.as_half2, z1);
    dq[1] = __hfma2(q1.as_half2, y16, z16);
    dq[2] = __hadd2(q2.as_half2, z1);
    dq[3] = __hfma2(q3.as_half2, y16, z16);
}

__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale
(
    const uint32_t zero,
    const half scale,
    half2 (&z1z16)[2],
    half2 (&y1y16)[2]
)
{
    half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
    half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));

    half2 scale2 = __half2half2(scale);

    z1z16[0] = __hmul2(scale2, __half2half2(z1.as_half));
    z1z16[1] = __hmul2(scale2, __half2half2(z16));

    const half y1 = __float2half_rn(1.0f);
    const half y16 = __float2half_rn(1.0f / 16.0f);

    y1y16[0] = __hmul2(scale2, __half2half2(y1));
    y1y16[1] = __hmul2(scale2, __half2half2(y16));
}

__forceinline__ __device__ void dequant_4bit_8_prep_zero
(
    const uint32_t zero,
    half2(&z1z16)[2],
    half2(&y1y16)[2]
)
{
    half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
    half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));

    z1z16[0] = __half2half2(z1.as_half);
    z1z16[1] = __half2half2(z16);

    const half y1 = __float2half_rn(1.0f);
    const half y16 = __float2half_rn(1.0f / 16.0f);

    y1y16[0] = __half2half2(y1);
    y1y16[1] = __half2half2(y16);
}


__forceinline__ __device__ void dequant_4bit_8_gptq
(
    const uint32_t q_0,
    half2 (&dq)[4],
    half2 (&z1z16)[2],
    half2 (&y1y16)[2],
    int stride,
    bool scaled
)
{
    const uint32_t c0 = 0x64006400;

    uint32_t qa = q_0;
    half2_uint32 q0((qa & 0x000f000f) | c0); // half2( q[0]      + 1024, q[1]      + 1024 )
    half2_uint32 q1((qa & 0x00f000f0) | c0); // half2( q[2] * 16 + 1024, q[3] * 16 + 1024 )
    qa >>= 8;
    half2_uint32 q2((qa & 0x000f000f) | c0); // half2( q[4]      + 1024, q[5]      + 1024 )
    half2_uint32 q3((qa & 0x00f000f0) | c0); // half2( q[6] * 16 + 1024, q[7] * 16 + 1024 )

    if (scaled)
    {
        dq[0] = __hfma2(q0.as_half2, y1y16[0], z1z16[0]);  // half2( q[0] * s - z * s, q[1] * s - z * s)
        dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]);  // half2( q[2] * s - z * s, q[3] * s - z * s)
        dq[2] = __hfma2(q2.as_half2, y1y16[0], z1z16[0]);
        dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]);
    }
    else
    {
        dq[0] = __hadd2(q0.as_half2,           z1z16[0]);  // half2( q[0] - z, q[1] - z )
        dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]);  // half2( q[2] - z, q[3] - z )
        dq[2] = __hadd2(q2.as_half2,           z1z16[0]);  // half2( q[4] - z, q[5] - z )
        dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]);  // half2( q[6] - z, q[7] - z )
    }
}

#else

__forceinline__ __device__ void shuffle_4bit_8
(
    uint32_t* q,
    int stride
)
{
}

__forceinline__ __device__ void dequant_4bit_8
(
    const uint32_t q_0,
    half2 (&dq)[4],
    int stride
)
{
    half dqh[8];
    for (int i = 0; i < 8; i++) dqh[i] = dq_ns(exb(q_0, i * 4, 0x0f), 8);

    for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
}

__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale
(
    const uint32_t zero,
    const half scale,
    half2 (&z1)[2],
    half2 (&y1)[2]
)
{
    half z = __int2half_rn(-((int)zero));
    z = __hmul(z, scale);
    z1[0] = __half2half2(z);
    y1[0] = __half2half2(scale);
}

__forceinline__ __device__ void dequant_4bit_8_prep_zero
(
    const uint32_t zero,
    half2(&z1)[2],
    half2(&y1)[2]
)
{
    half z = __int2half_rn(-((int)zero));
    z1[0] = __half2half2(z);
}

__forceinline__ __device__ void dequant_4bit_8_gptq
(
    const uint32_t q_0,
    half2 (&dq)[4],
    half2 (&z1)[2],
    half2 (&y1)[2],
    int stride,
    bool scaled
)
{
    half2 dqh2[8];

    uint32_t qa = q_0;
    for (int i = 0; i < 4; i++)
    {
        half d0 = __int2half_rn(qa & 0x0f); qa >>= 4;
        half d1 = __int2half_rn(qa & 0x0f); qa >>= 4;
        dqh2[i] = __halves2half2(d0, d1);
    }

    if (scaled)
    {
        dq[0] = __hfma2(dqh2[0], y1[0], z1[0]);
        dq[1] = __hfma2(dqh2[1], y1[0], z1[0]);
        dq[2] = __hfma2(dqh2[2], y1[0], z1[0]);
        dq[3] = __hfma2(dqh2[3], y1[0], z1[0]);
    }
    else
    {
        dq[0] = __hadd2(dqh2[0], z1[0]);
        dq[1] = __hadd2(dqh2[1], z1[0]);
        dq[2] = __hadd2(dqh2[2], z1[0]);
        dq[3] = __hadd2(dqh2[3], z1[0]);
    }
}

#endif

#endif

import gc
import torch
from torch import nn
import torch.nn.functional as F
import math
from exllamav2.ext import exllamav2_ext as ext_c, none_tensor
from exllamav2.util import list_live_tensors


class AdaptiveQuantizer:

    norm: float = 3.5
    max_p: float = 1.0
    min_p: float = 0.75
    p_grid: int = 48

    bits: int
    scale_bits: int
    scale_range: float = 1.0

    scale: torch.tensor
    qscale: torch.tensor
    qscale_max: float

    maxq: float
    scale_maxq: float
    qzero: float

    def __init__(self,
                 bits: int = 4,
                 scale_bits: int = 4):

        self.bits = bits
        self.scale_bits = scale_bits
        self.maxq = 2 ** bits - 1
        self.qzero = (self.maxq + 1) / 2
        self.scale_maxq = 2 ** scale_bits - 1

        self.scale_maxq = (2 ** self.scale_bits) - 1


    def find_params(self, x):

        xmax, _ = torch.max(torch.abs(x), dim = 0)
        xmax += 1e-12

        base_scale = xmax / (self.maxq / 2)
        qscale_max_t = torch.max(base_scale) * self.scale_range

        scale_tp = base_scale / qscale_max_t
        scale_tp = torch.sqrt(scale_tp)
        scale_tp *= (self.scale_maxq + 1)
        qscale_t = torch.clamp(torch.round(scale_tp), 1, self.scale_maxq + 1)
        qscale_tw = qscale_t / (self.scale_maxq + 1)
        qscale_tw = qscale_tw ** 2
        qscale_tw *= qscale_max_t

        q = torch.zeros((self.p_grid + 1, 128), dtype = torch.float, device = x.device)
        ext_c.quantize_err(x, q, qscale_tw, self.qzero, self.maxq, self.norm, self.min_p, self.max_p, self.p_grid)

        q = torch.sum(q, dim = 1)
        best_pi = torch.argmin(q)
        best_pif = best_pi / self.p_grid
        best_p = self.max_p * best_pif + self.min_p * (1 - best_pif)

        # best_p = 1.0

        self.qscale = qscale_t.to(torch.short)
        self.scale = qscale_tw * best_p
        self.qscale_max = qscale_max_t * best_p


class AdaptiveGPTQ:

    percdamp: float = 0.07

    layer: nn.Linear
    device: torch.device

    group_size: int
    bits: list
    bits_groups: list
    scale_bits: int
    hot_bits: int

    columns: int
    rows: int
    hessian: torch.tensor
    total_groups: int

    perm: torch.tensor = None
    invperm: torch.tensor = None

    g_idx: torch.tensor = None
    scale: torch.tensor = None
    qscale: torch.tensor = None
    qscale_max: torch.tensor = None
    qweight: torch.tensor = None
    qgroups: torch.tensor = None

    quant: torch.tensor = None
    weights: torch.tensor = None
    hessian: torch.tensor = None
    hessian_inv: torch.tensor = None
    num_samples: int = 0
    num_batches: int = 0


    def __init__(self,
                 layer: nn.Linear):

        self.layer = layer
        self.device = layer.weight.device

        self.rows = self.layer.weight.data.shape[1]
        self.columns = self.layer.weight.data.shape[0]

        self.weights = self.layer.weight.data.T.clone().float().contiguous()
        self.hessian = None
        self.num_samples = 0
        self.num_batches = 0


    def drop_buffers(self):

        self.perm = None
        self.invperm = None
        self.g_idx = None
        self.scale = None
        self.qscale = None
        self.qscale_max = None
        self.qweight = None
        self.qgroups = None
        self.quant = None
        self.weights = None
        self.hessian = None
        self.hessian_inv = None

        gc.collect()
        torch.cuda.empty_cache()


    def configure(self,
                  group_size: dict,
                  bits = None,
                  bits_prop = None,
                  scale_bits: int = 4
                  ):

        self.group_size = group_size
        self.scale_bits = scale_bits
        self.bits = bits

        assert isinstance(bits, list)
        assert isinstance(bits_prop, list)
        assert sum(bits_prop) == 1

        groups = 0
        remaining_rows = self.rows
        self.bits_groups = []
        for b, p in zip(self.bits, bits_prop):
            assert p > 0
            gsz = self.group_size[b]
            g = math.ceil(min(self.rows * p, remaining_rows) / gsz)
            groups += g
            remaining_rows -= g * gsz
            self.bits_groups.append(g)

        assert remaining_rows <= 0

        self.total_groups = groups

        # if isinstance(bits, list):
        #
        #     self.bits = bits
        #     g128 = (self.rows + 128 - 1) // 128
        #     self.bits_groups = [max(round(g128 * p), 1) * 128 // self.group_size for p in bits_prop]
        #     e = sum(self.bits_groups) - self.total_groups
        #     self.bits_groups[-1] -= e
        #
        # else:
        #
        #     self.bits = [bits]
        #     self.bits_groups = [self.total_groups]


    # def num_bits(self, subtract_columns = 0):
    #
    #     gi = self.g_idx.numel() * 32
    #     qs = self.qscale.numel() * self.scale_bits
    #     qss = self.qscale_max.numel() * 16
    #
    #     w = 0
    #     tr = self.rows
    #     for g, b in zip(self.bits_groups, self.bits):
    #
    #         c = self.columns - subtract_columns
    #         r = self.group_size * g
    #         if r > tr: r = tr
    #         tr -= r
    #         w += r * c * b
    #
    #     return w + gi + qs + qss


    def add_batch(self, inputs):

        with torch.inference_mode():

            if self.hessian is None:
                self.hessian = torch.zeros((self.rows, self.rows), device=self.device, dtype=torch.float)

            self.num_batches += 1
            num_samples = len(inputs)
            inputs = torch.cat(inputs, dim = 0)
            inputs = inputs.view((-1, inputs.shape[-1])).float().T.to("cuda:0")
            inputs *= math.sqrt(2 / num_samples)
            self.hessian += inputs.matmul(inputs.T)


    def prepare(self):

        with torch.inference_mode():

            self.hessian /= self.num_batches

            diagonal = torch.diag(self.hessian)

            # Zero weights that have no impact. Disabling this since it feels a little drastic based on just the calibration
            # data. It likely never triggers, anyway.

            # dead = diagonal == 0.0
            # self.hessian[dead, dead] = 1
            # self.weights[dead, :] = 0

            # Activation order

            self.perm = torch.argsort(diagonal, descending = True)
            self.weights = self.weights[self.perm, :]
            hessian = self.hessian[self.perm][:, self.perm]
            self.hessian = None

            # In case numerical errors have caused some asymmetry in H, assume it's close to symmetrical and force it.
            # (Doesn't seem to be needed)

            # torch.cuda.empty_cache()
            # hessian = (hessian + hessian.T) * 0.5
            # torch.cuda.empty_cache()

            # Damping

            diagonal = torch.diag(hessian)
            damp = torch.clamp(self.percdamp * torch.mean(diagonal), min = 1e-5)

            # Inverse of H

            attempts = 0
            while True:

                try:

                    d = torch.arange(self.rows, device = self.device)
                    hessian[d, d] += damp

                    # Dump condition number and smallest eigenvalue (should be positive)

                    # fro_norm_hessian = torch.norm(hessian, p = 'fro')
                    # fro_norm_inv = torch.norm(torch.linalg.inv(hessian), p = 'fro')
                    # cond_number = fro_norm_hessian * fro_norm_inv
                    # print(cond_number)

                    # eigenvalues = torch.linalg.eigvalsh(hessian)
                    # is_pd = torch.all(eigenvalues > 0)
                    # print(is_pd)
                    # print(torch.min(eigenvalues))

                    hessian_inv = torch.linalg.cholesky(hessian)
                    hessian_inv = torch.cholesky_inverse(hessian_inv)

                    # The Cholesky inverse will sometimes fail to compute due to accumulated rounding errors when H
                    # is very large (e.g. 70B MLP down proj) and a lot of calibration data is used (e.g. 100 rows of
                    # 4096 tokens). This won't always throw an exception and sometimes just results in a NaN tensor.

                    if torch.any(torch.isnan(hessian_inv)): raise RuntimeError

                    # Test inversion

                    # test = hessian_inv @ hessian
                    # test.sub_(torch.eye(test.size(0), device = test.device, dtype = test.dtype))
                    # test **= 2
                    # test = test.mean()
                    # print(test)

                    hessian_inv = torch.linalg.cholesky(hessian_inv, upper = True)
                    hessian_inv = hessian_inv.contiguous()

                    break

                except RuntimeError:

                    # If inverting failed, assume there were non-positive eigenvalues, so apply more damping to shift
                    # the eigenvalues in a positive direction.

                    print(" !! Warning: Applied additional damping")

                    attempts += 1
                    if attempts == 10:
                        raise ValueError("Hessian is not invertible")

            self.hessian_inv = hessian_inv
            self.hessian = None

    def reuse_h(self, other):

        with torch.inference_mode():

            self.hessian_inv = other.hessian_inv
            self.hessian = None
            self.perm = other.perm
            self.weights = self.weights[self.perm, :]


    def quantize(self, keep_qweight = False, apply = False, drop = False):

        with torch.inference_mode():

            if apply:
                weights = self.weights
                self.layer.weight.data = torch.zeros((1, 1), dtype = torch.float32, device = weights.device)
            else:
                weights = self.weights.clone()

            self.quant = torch.zeros_like(self.weights)

            if keep_qweight:
                self.qweight = torch.zeros_like(weights, dtype = torch.short)

            # Quantize groups

            scale = []
            qscale = []
            qscale_max = []
            qgroups = []

            error = weights.clone()
            group_idx = 0
            group_idx_list = []

            b = 0
            for bits_idx, bits in enumerate(self.bits):
                quantizer = AdaptiveQuantizer(bits = bits, scale_bits = self.scale_bits)

                for group in range(self.bits_groups[bits_idx]):
                    a = b
                    b = min(a + self.group_size[bits], self.rows)

                    qgroups.append(bits)
                    qgroups.append(0)

                    quantizer.find_params(weights[a : b, :])
                    scale.append(quantizer.scale)
                    qscale.append(quantizer.qscale)
                    qscale_max.append(quantizer.qscale_max)

                    ext_c.quantize_range(self.quant,
                                         quantizer.scale,
                                         self.qweight if keep_qweight else none_tensor,
                                         quantizer.qzero,
                                         quantizer.maxq,
                                         self.hessian_inv,
                                         weights,
                                         error,
                                         a,
                                         b)

                    group_idx_list += [group_idx] * (b - a)
                    group_idx += 1


            # Create g_idx to store inverse activation order

            self.g_idx = torch.tensor(group_idx_list, dtype = torch.int32, device = self.device)

            self.invperm = torch.argsort(self.perm)
            self.g_idx = self.g_idx[self.invperm]

            # Store scales

            self.scale = torch.stack(scale, dim = 0)
            self.qscale = torch.stack(qscale, dim = 0)
            self.qscale_max = torch.tensor(qscale_max, dtype = torch.float16, device = self.device)
            self.qgroups = torch.tensor(qgroups, dtype = torch.short, device = self.device)

            # Apply

            if apply:
                if drop:
                    weights = None
                    error = None
                    scale = None
                    qscale = None
                    qscale_max = None
                    qgroups = None
                    group_idx_list = None
                    gc.collect()
                    torch.cuda.empty_cache()
                self.apply_quant()


    def quant_error(self):

        with torch.inference_mode():

            q = self.quant[self.invperm, :]
            diff = torch.abs(q - self.layer.weight.data.T)
            mat_error_1 = (diff > 0.01).sum().item() / diff.numel()
            mat_error_5 = (diff > 0.05).sum().item() / diff.numel()
            mat_error_10 = (diff > 0.10).sum().item() / diff.numel()
            return mat_error_1, mat_error_5, mat_error_10


    def apply_quant(self):

        qc = self.quant.cpu()
        invperm = self.invperm.cpu()
        q = qc[invperm, :].T
        q = q.reshape(self.quant.T.shape)
        q = q.to(self.quant.device)
        self.layer.weight.data = q


    def apply_temp(self):

        q = self.quant[self.invperm, :].T
        temp_layer = nn.Linear(self.layer.in_features, self.layer.out_features, False, device = "meta", dtype = torch.float16)
        temp_layer.weight = nn.Parameter(q.reshape(self.layer.weight.shape).type_as(self.layer.weight.data))
        return temp_layer


    def pack(self, key, qparams):

        assert qparams.scale_bits in [4]
        # assert self.columns % 32 == 0

        output = {}
        output[key + ".q_invperm"] = self.invperm.to(torch.int)
        output[key + ".q_scale_max"] = self.qscale_max
        output[key + ".q_groups"] = self.qgroups

        columns = self.columns
        rem_rows = self.rows
        padding = -columns % 32

        if padding != 0:
            print(f" !! Note: Padding quantized tensor {key}")
            qst = F.pad(self.qscale, (0, padding)).contiguous()
            qwt = F.pad(self.qweight, (0, padding)).contiguous()
        else:
            qst = self.qscale
            qwt = self.qweight

        qst_packed = torch.zeros((qst.shape[0], qst.shape[1] * qparams.scale_bits // 32), dtype = torch.int32, device = self.device)
        if qparams.scale_bits == 4: ext_c.pack_rows_4(qst, qst_packed)
        # if qparams.scale_bits == 6: ext_c.pack_rows_6(qst, qst_packed) # TODO:
        output[key + ".q_scale"] = qst_packed

        qwt_packed = []

        i = 0
        row = 0
        out_row = 0
        while i < self.qscale.shape[0]:

            bits = self.qgroups[i * 2].item()
            self.qgroups[i * 2 + 1] = out_row
            i += 1

            rows = min(self.group_size[bits], rem_rows)
            wpqr = 32 / bits
            qrows = rows / wpqr
            assert i == self.qgroups.shape[-1] or qrows == int(qrows)
            qrows = math.ceil(qrows)

            g_qwt = qwt[row:row+rows, :].contiguous()
            g_qwt_packed = torch.zeros((qrows, columns + padding), dtype = torch.int32, device = self.device)

            if padding > 0: g_qwt[:, -padding:] = 2 ** (bits - 1)

            ext_c.pack_columns(g_qwt, g_qwt_packed, bits)
            qwt_packed.append(g_qwt_packed)

            # print(row, rows, bits)

            row += rows
            out_row += qrows
            rem_rows -= rows


        qwt_packed = torch.cat(qwt_packed, dim = 0)
        output[key + ".q_weight"] = qwt_packed

        return output


This page shows examples of messages formatted using JSON (JavaScript Object Notation).

{
    "glossary": {
        "title": "example glossary",
		"GlossDiv": {
            "title": "S",
			"GlossList": {
                "GlossEntry": {
                    "ID": "SGML",
					"SortAs": "SGML",
					"GlossTerm": "Standard Generalized Markup Language",
					"Acronym": "SGML",
					"Abbrev": "ISO 8879:1986",
					"GlossDef": {
                        "para": "A meta-markup language, used to create markup languages such as DocBook.",
						"GlossSeeAlso": ["GML", "XML"]
                    },
					"GlossSee": "markup"
                }
            }
        }
    }
}

The same text expressed as XML:

<!DOCTYPE glossary PUBLIC "-//OASIS//DTD DocBook V3.1//EN">
 <glossary><title>example glossary</title>
  <GlossDiv><title>S</title>
   <GlossList>
    <GlossEntry ID="SGML" SortAs="SGML">
     <GlossTerm>Standard Generalized Markup Language</GlossTerm>
     <Acronym>SGML</Acronym>
     <Abbrev>ISO 8879:1986</Abbrev>
     <GlossDef>
      <para>A meta-markup language, used to create markup
languages such as DocBook.</para>
      <GlossSeeAlso OtherTerm="GML">
      <GlossSeeAlso OtherTerm="XML">
     </GlossDef>
     <GlossSee OtherTerm="markup">
    </GlossEntry>
   </GlossList>
  </GlossDiv>
 </glossary>

{"menu": {
  "id": "file",
  "value": "File",
  "popup": {
    "menuitem": [
      {"value": "New", "onclick": "CreateNewDoc()"},
      {"value": "Open", "onclick": "OpenDoc()"},
      {"value": "Close", "onclick": "CloseDoc()"}
    ]
  }
}}

The same text expressed as XML:

<menu id="file" value="File">
  <popup>
    <menuitem value="New" onclick="CreateNewDoc()" />
    <menuitem value="Open" onclick="OpenDoc()" />
    <menuitem value="Close" onclick="CloseDoc()" />
  </popup>
</menu>

{"widget": {
    "debug": "on",
    "window": {
        "title": "Sample Konfabulator Widget",
        "name": "main_window",
        "width": 500,
        "height": 500
    },
    "image": {
        "src": "Images/Sun.png",
        "name": "sun1",
        "hOffset": 250,
        "vOffset": 250,
        "alignment": "center"
    },
    "text": {
        "data": "Click Here",
        "size": 36,
        "style": "bold",
        "name": "text1",
        "hOffset": 250,
        "vOffset": 100,
        "alignment": "center",
        "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;"
    }
}}

The same text expressed as XML:

<widget>
    <debug>on</debug>
    <window title="Sample Konfabulator Widget">
        <name>main_window</name>
        <width>500</width>
        <height>500</height>
    </window>
    <image src="Images/Sun.png" name="sun1">
        <hOffset>250</hOffset>
        <vOffset>250</vOffset>
        <alignment>center</alignment>
    </image>
    <text data="Click Here" size="36" style="bold">
        <name>text1</name>
        <hOffset>250</hOffset>
        <vOffset>100</vOffset>
        <alignment>center</alignment>
        <onMouseUp>
            sun1.opacity = (sun1.opacity / 100) * 90;
        </onMouseUp>
    </text>
</widget>

{"web-app": {
  "servlet": [
    {
      "servlet-name": "cofaxCDS",
      "servlet-class": "org.cofax.cds.CDSServlet",
      "init-param": {
        "configGlossary:installationAt": "Philadelphia, PA",
        "configGlossary:adminEmail": "ksm@pobox.com",
        "configGlossary:poweredBy": "Cofax",
        "configGlossary:poweredByIcon": "/images/cofax.gif",
        "configGlossary:staticPath": "/content/static",
        "templateProcessorClass": "org.cofax.WysiwygTemplate",
        "templateLoaderClass": "org.cofax.FilesTemplateLoader",
        "templatePath": "templates",
        "templateOverridePath": "",
        "defaultListTemplate": "listTemplate.htm",
        "defaultFileTemplate": "articleTemplate.htm",
        "useJSP": false,
        "jspListTemplate": "listTemplate.jsp",
        "jspFileTemplate": "articleTemplate.jsp",
        "cachePackageTagsTrack": 200,
        "cachePackageTagsStore": 200,
        "cachePackageTagsRefresh": 60,
        "cacheTemplatesTrack": 100,
        "cacheTemplatesStore": 50,
        "cacheTemplatesRefresh": 15,
        "cachePagesTrack": 200,
        "cachePagesStore": 100,
        "cachePagesRefresh": 10,
        "cachePagesDirtyRead": 10,
        "searchEngineListTemplate": "forSearchEnginesList.htm",
        "searchEngineFileTemplate": "forSearchEngines.htm",
        "searchEngineRobotsDb": "WEB-INF/robots.db",
        "useDataStore": true,
        "dataStoreClass": "org.cofax.SqlDataStore",
        "redirectionClass": "org.cofax.SqlRedirection",
        "dataStoreName": "cofax",
        "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver",
        "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon",
        "dataStoreUser": "sa",
        "dataStorePassword": "dataStoreTestQuery",
        "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';",
        "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log",
        "dataStoreInitConns": 10,
        "dataStoreMaxConns": 100,
        "dataStoreConnUsageLimit": 100,
        "dataStoreLogLevel": "debug",
        "maxUrlLength": 500}},
    {
      "servlet-name": "cofaxEmail",
      "servlet-class": "org.cofax.cds.EmailServlet",
      "init-param": {
      "mailHost": "mail1",
      "mailHostOverride": "mail2"}},
    {
      "servlet-name": "cofaxAdmin",
      "servlet-class": "org.cofax.cds.AdminServlet"},

    {
      "servlet-name": "fileServlet",
      "servlet-class": "org.cofax.cds.FileServlet"},
    {
      "servlet-name": "cofaxTools",
      "servlet-class": "org.cofax.cms.CofaxToolsServlet",
      "init-param": {
        "templatePath": "toolstemplates/",
        "log": 1,
        "logLocation": "/usr/local/tomcat/logs/CofaxTools.log",
        "logMaxSize": "",
        "dataLog": 1,
        "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log",
        "dataLogMaxSize": "",
        "removePageCache": "/content/admin/remove?cache=pages&id=",
        "removeTemplateCache": "/content/admin/remove?cache=templates&id=",
        "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder",
        "lookInContext": 1,
        "adminGroupID": 4,
        "betaServer": true}}],
  "servlet-mapping": {
    "cofaxCDS": "/",
    "cofaxEmail": "/cofaxutil/aemail/*",
    "cofaxAdmin": "/admin/*",
    "fileServlet": "/static/*",
    "cofaxTools": "/tools/*"},

  "taglib": {
    "taglib-uri": "cofax.tld",
    "taglib-location": "/WEB-INF/tlds/cofax.tld"}}}

The same file expressed as XML:

<?xml version="1.0" encoding="ISO-8859-1"?>

<!DOCTYPE web-app
    PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.2//EN"
    "http://java.sun.com/j2ee/dtds/web-app_2.2.dtd">
<web-app>
    <servlet>
        <servlet-name>
            cofaxCDS
        </servlet-name>
        <servlet-class>
            org.cofax.cds.CDSServlet
        </servlet-class>

         <init-param>
          <param-name>configGlossary:installationAt</param-name>
      <param-value>Philadelphia, PA</param-value>
        </init-param>
        <init-param>
          <param-name>configGlossary:adminEmail</param-name>
      <param-value>ksm@pobox.com</param-value>
        </init-param>
        <init-param>
          <param-name>configGlossary:poweredBy</param-name>
      <param-value>Cofax</param-value>
        </init-param>
        <init-param>
          <param-name>configGlossary:poweredByIcon</param-name>
      <param-value>/images/cofax.gif</param-value>
        </init-param>
        <init-param>
          <param-name>configGlossary:staticPath</param-name>
      <param-value>/content/static</param-value>
        </init-param>

     <init-param>
          <param-name>templateProcessorClass</param-name>
      <param-value>org.cofax.WysiwygTemplate</param-value>
        </init-param>

    <init-param>
          <param-name>templateLoaderClass</param-name>
      <param-value>org.cofax.FilesTemplateLoader</param-value>
        </init-param>

         <init-param>
          <param-name>templatePath</param-name>
      <param-value>templates</param-value>
        </init-param>

        <init-param>
          <param-name>templateOverridePath</param-name>
      <param-value></param-value>
        </init-param>

     <init-param>
          <param-name>defaultListTemplate</param-name>
      <param-value>listTemplate.htm</param-value>
        </init-param>

       <init-param>
          <param-name>defaultFileTemplate</param-name>
      <param-value>articleTemplate.htm</param-value>
        </init-param>

       <init-param>
          <param-name>useJSP</param-name>
      <param-value>false</param-value>
        </init-param>

    <init-param>
          <param-name>jspListTemplate</param-name>
      <param-value>listTemplate.jsp</param-value>
        </init-param>

       <init-param>
          <param-name>jspFileTemplate</param-name>
      <param-value>articleTemplate.jsp</param-value>
        </init-param>

     <init-param>
          <param-name>cachePackageTagsTrack</param-name>
      <param-value>200</param-value>
        </init-param>

    <init-param>
          <param-name>cachePackageTagsStore</param-name>
      <param-value>200</param-value>
        </init-param>

    <init-param>
          <param-name>cachePackageTagsRefresh</param-name>
      <param-value>60</param-value>
        </init-param>

     <init-param>
          <param-name>cacheTemplatesTrack</param-name>
      <param-value>100</param-value>
        </init-param>

    <init-param>
          <param-name>cacheTemplatesStore</param-name>
      <param-value>50</param-value>
        </init-param>

    <init-param>
          <param-name>cacheTemplatesRefresh</param-name>
      <param-value>15</param-value>
        </init-param>

    <init-param>
          <param-name>cachePagesTrack</param-name>
      <param-value>200</param-value>
        </init-param>

    <init-param>
          <param-name>cachePagesStore</param-name>
      <param-value>100</param-value>
        </init-param>

    <init-param>
          <param-name>cachePagesRefresh</param-name>
      <param-value>10</param-value>
        </init-param>

    <init-param>
          <param-name>cachePagesDirtyRead</param-name>
      <param-value>10</param-value>
        </init-param>

     <init-param>
          <param-name>searchEngineListTemplate</param-name>
      <param-value>forSearchEnginesList.htm</param-value>
        </init-param>

       <init-param>
          <param-name>searchEngineFileTemplate</param-name>
      <param-value>forSearchEngines.htm</param-value>
        </init-param>

       <init-param>
          <param-name>searchEngineRobotsDb</param-name>
      <param-value>WEB-INF/robots.db</param-value>
        </init-param>

    <init-param>
          <param-name>useDataStore</param-name>
      <param-value>true</param-value>
        </init-param>

     <init-param>
          <param-name>dataStoreClass</param-name>
      <param-value>org.cofax.SqlDataStore</param-value>
        </init-param>

        <init-param>
          <param-name>redirectionClass</param-name>
      <param-value>org.cofax.SqlRedirection</param-value>
        </init-param>
        <init-param>
          <param-name>dataStoreName</param-name>
      <param-value>cofax</param-value>
        </init-param>

        <init-param>
          <param-name>dataStoreDriver</param-name>
      <param-value>com.microsoft.jdbc.sqlserver.SQLServerDriver</param-value>
        </init-param>

         <init-param>
          <param-name>dataStoreUrl</param-name>
      <param-value>jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon</param-value>
        </init-param>

        <init-param>
          <param-name>dataStoreUser</param-name>
      <param-value>sa</param-value>
        </init-param>

         <init-param>
          <param-name>dataStorePassword</param-name>
      <param-value></param-value>
        </init-param>

         <init-param>
          <param-name>dataStoreTestQuery</param-name>
      <param-value>SET NOCOUNT ON;select test='test';</param-value>
        </init-param>

         <init-param>
          <param-name>dataStoreLogFile</param-name>
      <param-value>/usr/local/tomcat/logs/datastore.log</param-value>
        </init-param>

         <init-param>
          <param-name>dataStoreInitConns</param-name>
      <param-value>10</param-value>
        </init-param>

         <init-param>
          <param-name>dataStoreMaxConns</param-name>
      <param-value>100</param-value>
        </init-param>

         <init-param>
          <param-name>dataStoreConnUsageLimit</param-name>
      <param-value>100</param-value>
        </init-param>
 <init-param>
          <param-name>dataStoreLogLevel</param-name>
      <param-value>debug</param-value>
        </init-param>

     <init-param>
          <param-name>maxUrlLength</param-name>
      <param-value>500</param-value>
        </init-param>

    </servlet>

     <servlet>
        <servlet-name>
            cofaxEmail
        </servlet-name>
        <servlet-class>
            org.cofax.cds.EmailServlet
        </servlet-class>

         <init-param>
          <param-name>mailHost</param-name>
      <param-value>mail1</param-value>
        </init-param>

        <init-param>
          <param-name>mailHostOverride</param-name>
      <param-value>mail2</param-value>
        </init-param>
    </servlet>

    <servlet>
        <servlet-name>
            cofaxAdmin
        </servlet-name>
        <servlet-class>
            org.cofax.cds.AdminServlet
        </servlet-class>
    </servlet>

    <servlet>
         <servlet-name>
             fileServlet
         </servlet-name>
         <servlet-class>
             org.cofax.cds.FileServlet
         </servlet-class>
    </servlet>

   <servlet>
        <servlet-name>
            cofaxTools
        </servlet-name>
        <servlet-class>
            org.cofax.cms.CofaxToolsServlet
        </servlet-class>

         <init-param>
          <param-name>templatePath</param-name>
          <param-value>toolstemplates/</param-value>
        </init-param>

         <init-param>
          <param-name>log</param-name>
          <param-value>1</param-value>
        </init-param>

         <init-param>
          <param-name>logLocation</param-name>
          <param-value>/usr/local/tomcat/logs/CofaxTools.log</param-value>
        </init-param>

         <init-param>
          <param-name>logMaxSize</param-name>
          <param-value></param-value>
        </init-param>

         <init-param>
          <param-name>dataLog</param-name>
          <param-value>1</param-value>
        </init-param>

         <init-param>
          <param-name>dataLogLocation</param-name>
          <param-value>/usr/local/tomcat/logs/dataLog.log</param-value>
        </init-param>

         <init-param>
          <param-name>dataLogMaxSize</param-name>
          <param-value></param-value>
        </init-param>

         <init-param>
          <param-name>removePageCache</param-name>
          <param-value>/content/admin/remove?cache=pages&id=</param-value>
        </init-param>

         <init-param>
          <param-name>removeTemplateCache</param-name>
          <param-value>/content/admin/remove?cache=templates&id=</param-value>
        </init-param>

         <init-param>
          <param-name>fileTransferFolder</param-name>
          <param-value>/usr/local/tomcat/webapps/content/fileTransferFolder</param-value>
        </init-param>

         <init-param>
          <param-name>lookInContext</param-name>
          <param-value>1</param-value>
        </init-param>

        <init-param>
          <param-name>adminGroupID</param-name>
          <param-value>4</param-value>
        </init-param>

        <init-param>
          <param-name>betaServer</param-name>
          <param-value>true</param-value>
        </init-param>

    </servlet>

    <servlet-mapping>
    <servlet-name>
    cofaxCDS
    </servlet-name>
    <url-pattern>
    /
    </url-pattern>
    </servlet-mapping>

    <servlet-mapping>
    <servlet-name>
    cofaxEmail
    </servlet-name>
    <url-pattern>
    /cofaxutil/aemail/*
    </url-pattern>
    </servlet-mapping>

    <servlet-mapping>
        <servlet-name>
        cofaxAdmin
        </servlet-name>
        <url-pattern>
        /admin/*
        </url-pattern>
    </servlet-mapping>

    <servlet-mapping>
         <servlet-name>
         fileServlet
         </servlet-name>
         <url-pattern>
         /static/*
         </url-pattern>
     </servlet-mapping>

  <servlet-mapping>
        <servlet-name>
        cofaxTools
        </servlet-name>
        <url-pattern>
        /tools/*
        </url-pattern>
    </servlet-mapping>

   <taglib>
      <taglib-uri>cofax.tld</taglib-uri>
      <taglib-location>/WEB-INF/tlds/cofax.tld</taglib-location>
   </taglib>

</web-app>


The action and label values only need to be provided if they are not the same as the id.

{"menu": {
    "header": "SVG Viewer",
    "items": [
        {"id": "Open"},
        {"id": "OpenNew", "label": "Open New"},
        null,
        {"id": "ZoomIn", "label": "Zoom In"},
        {"id": "ZoomOut", "label": "Zoom Out"},
        {"id": "OriginalView", "label": "Original View"},
        null,
        {"id": "Quality"},
        {"id": "Pause"},
        {"id": "Mute"},
        null,
        {"id": "Find", "label": "Find..."},
        {"id": "FindAgain", "label": "Find Again"},
        {"id": "Copy"},
        {"id": "CopyAgain", "label": "Copy Again"},
        {"id": "CopySVG", "label": "Copy SVG"},
        {"id": "ViewSVG", "label": "View SVG"},
        {"id": "ViewSource", "label": "View Source"},
        {"id": "SaveAs", "label": "Save As"},
        null,
        {"id": "Help"},
        {"id": "About", "label": "About Adobe CVG Viewer..."}
    ]
}}

The same message expressed as XML:

<menu>
    <header>Adobe SVG Viewer</header>
    <item action="Open" id="Open">Open</item>
    <item action="OpenNew" id="OpenNew">Open New</item>
    <separator/>
    <item action="ZoomIn" id="ZoomIn">Zoom In</item>
    <item action="ZoomOut" id="ZoomOut">Zoom Out</item>
    <item action="OriginalView" id="OriginalView">Original View</item>
    <separator/>
    <item action="Quality" id="Quality">Quality</item>
    <item action="Pause" id="Pause">Pause</item>
    <item action="Mute" id="Mute">Mute</item>
    <separator/>
    <item action="Find" id="Find">Find...</item>
    <item action="FindAgain" id="FindAgain">Find Again</item>
    <item action="Copy" id="Copy">Copy</item>
    <item action="CopyAgain" id="CopyAgain">Copy Again</item>
    <item action="CopySVG" id="CopySVG">Copy SVG</item>
    <item action="ViewSVG" id="ViewSVG">View SVG</item>
    <item action="ViewSource" id="ViewSource">View Source</item>
    <item action="SaveAs" id="SaveAs">Save As</item>
    <separator/>
    <item action="Help" id="Help">Help</item>
    <item action="About" id="About">About Adobe CVG Viewer...</item>
</menu>

We have discussed qsort() in C. C++ STL provides a similar function sort that sorts a vector or array (items with random access)

It generally takes two parameters, the first one being the point of the array/vector from where the sorting needs to begin and the second parameter being the length up to which we want the array/vector to get sorted. The third parameter is optional and can be used in cases such as if we want to sort the elements lexicographically.

By default, the sort() function sorts the elements in ascending order.

Below is a simple program to show the working of sort().

// C++ program to demonstrate default behaviour of
// sort() in STL.
#include <bits/stdc++.h>
using namespace std;

int main()
{
    int arr[] = { 1, 5, 8, 9, 6, 7, 3, 4, 2, 0 };
    int n = sizeof(arr) / sizeof(arr[0]);

    /*Here we take two parameters, the beginning of the
    array and the length n upto which we want the array to
    be sorted*/
    sort(arr, arr + n);

    cout << "\nArray after sorting using "
            "default sort is : \n";
    for (int i = 0; i < n; ++i)
        cout << arr[i] << " ";

    return 0;
}
Output

Array after sorting using default sort is :
0 1 2 3 4 5 6 7 8 9

Time Complexity: O(N log N)
Auxiliary Space: O(1)

How to sort in descending order?
sort() takes a third parameter that is used to specify the order in which elements are to be sorted. We can pass the “greater()” function to sort in descending order. This function does a comparison in a way that puts greater elements before.

// C++ program to demonstrate descending order sort using
// greater<>().
#include <bits/stdc++.h>
using namespace std;

int main()
{
    int arr[] = { 1, 5, 8, 9, 6, 7, 3, 4, 2, 0 };
    int n = sizeof(arr) / sizeof(arr[0]);

    sort(arr, arr + n, greater<int>());

    cout << "Array after sorting : \n";
    for (int i = 0; i < n; ++i)
        cout << arr[i] << " ";

    return 0;
}
Output

Array after sorting :
9 8 7 6 5 4 3 2 1 0

Time Complexity: O(N log N)
Auxiliary Space: O(1)

Sort the array only in the given range: To deal with such types of problems we just have to mention the range inside the sort function.
Below is the implementation of above case:

// C++ program to demonstrate sort()

#include <bits/stdc++.h>
using namespace std;

int main()
{
    int arr[] = { 0, 1, 5, 8, 9, 6, 7, 3, 4, 2 };
    int n = sizeof(arr) / sizeof(arr[0]);

    // Sort the elements which lies in the range of 2 to
    // (n-1)
    sort(arr + 2, arr + n);

    cout << "Array after sorting : \n";
    for (int i = 0; i < n; ++i)
        cout << arr[i] << " ";

    return 0;
}
// This code is contributed by Suruchi Kumari
Output

Array after sorting :
0 1 2 3 4 5 6 7 8 9

Time Complexity: O(N log N)

Auxiliary Space: O(1)

How to sort in a particular order?
We can also write our own comparator function and pass it as a third parameter. This “comparator” function returns a value; convertible to bool, which basically tells us whether the passed “first” argument should be placed before the passed “second” argument or not.
For eg: In the code below, suppose intervals {6,8} and {1,9} are passed as arguments in the “compareInterval” function(comparator function). Now as i1.first (=6) < i2.first (=1), so our function returns “false”, which tells us that “first” argument should not be placed before “second” argument and so sorting will be done in order like {1,9} first and then {6,8} as next.

// A C++ program to demonstrate
// STL sort() using
// our own comparator
#include <bits/stdc++.h>
using namespace std;

// An interval has a start
// time and end time
struct Interval {
    int start, end;
};

// Compares two intervals
// according to starting times.
bool compareInterval(Interval i1, Interval i2)
{
    return (i1.start < i2.start);
}

int main()
{
    Interval arr[]
        = { { 6, 8 }, { 1, 9 }, { 2, 4 }, { 4, 7 } };
    int n = sizeof(arr) / sizeof(arr[0]);

    // sort the intervals in increasing order of
    // start time
    sort(arr, arr + n, compareInterval);

    cout << "Intervals sorted by start time : \n";
    for (int i = 0; i < n; i++)
        cout << "[" << arr[i].start << "," << arr[i].end
             << "] ";

    return 0;
}
Output

Intervals sorted by start time :
[1,9] [2,4] [4,7] [6,8]

The time complexity of std::sort() is:

    Best Case – O(N log N)
    Average Case – O(N log N)
    Worst-Case – O(N log N)

Space Complexity: It may use O( log N) auxiliary space.

#include <algorithm>
#include <iostream>
using namespace std;

template <class T>
class Comparator { // we pass an object of this class as
                   // third arg to sort function...
public:
    bool operator()(T x1, T x2)
    {
        return x1 < x2;
    }
};

template <class T> bool funComparator(T x1, T x2)
{ // return type is bool
    return x1 <= x2;
}

void show(int a[], int array_size)
{
    for (int i = 0; i < array_size; i++) {
        cout << a[i] << " ";
    }
}

int main()
{
    int a[] = { 1, 5, 8, 9, 6, 7, 3, 4, 2, 0 };
    int asize = sizeof(a) / sizeof(int);
    cout << "The array before sorting is : ";
    show(a, asize);
    cout << endl << "The array after sorting is(asc) :";
    sort(a, a + asize);
    show(a, asize);
    cout << endl << "The array after sorting is(desc) :";
    sort(a, a + asize, greater<int>());
    show(a, asize);
    cout << endl
         << "The array after sorting is(asc but our "
            "comparator class) :";
    sort(a, a + asize, Comparator<int>());
    show(a, asize);
    cout << endl
         << "The array after sorting is(asc but our "
            "comparator function) :";
    sort(a, a + asize, funComparator<int>);
    show(a, asize);

    return 0;
}
Output

The array before sorting is : 1 5 8 9 6 7 3 4 2 0
The array after sorting is(asc) :0 1 2 3 4 5 6 7 8 9
The array after sorting is(desc) :9 8 7 6 5 4 3 2 1 0
The array after sorting is(asc but our comparator class) :0 1 2 3 4 5 6 7 8 9
The array after sorting is(asc but our comparator function) :0 1 2 3 4 5 6 7 8 9

Time Complexity: O(N log N)
Auxiliary Space: O(1)

Please write comments if you find anything incorrect, or you want to share more information about the topic discussed above

The following are different ways to construct or initialize a vector in C++ STL

1. Initializing by pushing values one by one:

// C++ program to create an empty
// vector and push values one
// by one.
#include <iostream>
#include <vector>
using namespace std;

int main()
{
    // Create an empty vector
    vector<int> vect;

    vect.push_back(10);
    vect.push_back(20);
    vect.push_back(30);

    for (int x : vect)
        cout << x << " ";

    return 0;
}
Output

10 20 30

2. Specifying size and initializing all values:

// C++ program to create an empty
// vector and push values one
// by one.
#include <iostream>
#include <vector>
using namespace std;

int main()
{
    int n = 3;

    // Create a vector of size n with
    // all values as 10.
    vector<int> vect(n, 10);

    for (int x : vect)
        cout << x << " ";

    return 0;
}
Output

10 10 10

3. Initializing like arrays:


// C++ program to initialize
// a vector like an array.
#include <iostream>
#include <vector>
using namespace std;

int main()
{
    vector<int> vect{ 10, 20, 30 };

    for (int x : vect)
        cout << x << " ";

    return 0;
}
Output

10 20 30

4. Initializing from an array:

// C++ program to initialize
// a vector from an array.
#include <iostream>
#include <vector>
using namespace std;

int main()
{
    int arr[] = { 10, 20, 30 };
    int n = sizeof(arr) / sizeof(arr[0]);

    vector<int> vect(arr, arr + n);

    for (int x : vect)
        cout << x << " ";

    return 0;
}
Output

10 20 30

5. Initializing from another vector:

// C++ program to initialize a vector from
// another vector.
#include <iostream>
#include <vector>
using namespace std;

int main()
{
    vector<int> vect1{ 10, 20, 30 };

    vector<int> vect2(vect1.begin(), vect1.end());

    for (int x : vect2)
        cout << x << " ";

    return 0;
}
Output

10 20 30

6. Initializing all elements with a particular value:

// C++ Program to initialize vector using fill()
#include <iostream>
#include <vector>
using namespace std;

int main()
{
    // creating array with size 10
    vector<int> vect1(10);

    // initializing using fill() function
    int value = 5;
    fill(vect1.begin(), vect1.end(), value);

    // printing vector
    for (int x : vect1)
        cout << x << " ";

    return 0;
}
Output

5 5 5 5 5 5 5 5 5 5

7. Initialize an array with consecutive numbers using std::iota:

// C++ program to initialize a
// vector with consecutive
// numbers
#include <iostream>
#include <numeric>
#include <vector>
using namespace std;

int main()
{
    // declaring a vector with size 5
    vector<int> vec(5);

    // initializing using iota()
    iota(vec.begin(), vec.end(), 1);

    // printing the vector
    for (int i = 0; i < 5; i++) {
        cout << vec[i] << " ";
    }
    return 0;
}
Output

1 2 3 4 5

Time complexity: O(N), where N is the size of the vector.

Auxiliary space: O(N).

If you like GeeksforGeeks and would like to contribute, you can also write an article using write.geeksforgeeks.org. See your article appearing on the GeeksforGeeks main page and help other Geeks. Please write comments if you find anything incorrect, or if you want to share more information about the topic discussed above.

Whether you're preparing for your first job interview or aiming to upskill in this ever-evolving tech landscape, GeeksforGeeks Courses are your key to success. We provide top-quality content at affordable prices, all geared towards accelerating your growth in a time-bound manner. Join the millions we've already empowered, and we're here to do the same for you. Don't miss out - check it out now!


How to Hide and Show a Console Window in C++?
Read
Discuss
Courses
Practice

The task is to hide and Show the console window of a C++ program. The program for the same is given below.

    Note: The results of the following program can only be seen when it is executed on a console.

Example:

// C++ program to hide and show a console window
#include <iostream>
#include <windows.h>

using namespace std;

void countdown()
{
    cout << "3" << endl;
    Sleep(1000);
    cout << "2" << endl;
    Sleep(1000);
    cout << "1" << endl;
    Sleep(1000);
    cout << "0" << endl;
}

int main()
{
    countdown();
    HWND window;
    AllocConsole();
    // You Can Find HANDLE of other windows too
    window = FindWindowA("ConsoleWindowClass", NULL);
    ShowWindow(window, 0);

    countdown();
    ShowWindow(window, 1);
}

Output:



Explanation: The above program counts from 3 to 1 before the Console Window disappears. After the window has disappeared, the ShowWindow helps the program so that the Console Window reappears again after counting from 3 to 1(executing the countdown function).

The execution of the program can be understood by understanding the key functions of the program.

    #include<windows.h> – The windows.h header in C++ programming languages are specifically designed for windows and contain a very large number of windows specific functions.
    AllocConsole()- AllocConsole initializes standard input, standard output, and standard error handles for the new console.
    ShowWindow()- Sets the specified window’s show state.
    FindWindowA()– Takes string parameters and checks whose class name and window name match the specified strings

Whether you're preparing for your first job interview or aiming to upskill in this ever-evolving tech landscape, GeeksforGeeks Courses are your key to success. We provide top-quality content at affordable prices, all geared towards accelerating your growth in a time-bound manner. Join the millions we've already empowered, and we're here to do the same for you. Don't miss out - check it out now!

Last Updated : 27 Nov, 2022

Prerequisite: Templates in C++

Substitution Failure Is Not An Error (SFINAE) is a principle of the C++ programming language that states that a compiler should not fail to compile a program just because it cannot substitute a template parameter. This principle allows for the use of template metaprogramming and enables the compiler to make decisions based on the types of template arguments, which can be useful when dealing with complex code and difficult to reason about logic.

At its core, SFINAE is a way of allowing a compiler to decide which template to use in a given context. The decision is based on the types of template arguments and the compiler will choose the template that is most appropriate for the arguments.
Advantages of SFINAE:

    SFINAE is useful in a variety of scenarios, including writing generic code and dealing with complex logic.
    SFINAE allows the compiler to decide which template to use, it allows programmers to write code that can be reused in different contexts without needing to explicitly specify the types of template parameters.
    SFINAE allows for better code reuse, as the same code can be used for different types of objects and parameters.
    SFINAE also allows for better control over code complexity by allowing the compiler to make decisions based on the types of template parameters, it can reduce the amount of complex logic that needs to be written and understood.
    SFINAE helps to make code more readable and maintainable, as it is easier to follow the logic of the program.

In conclusion, SFINAE is a powerful concept in C++ programming that allows for better code reuse, improved code readability, and better control over code complexity. It is a key component of template metaprogramming and is an essential tool for writing robust and efficient code.

Example:

// C++ Program to implement
// Substitution Failure Is Not An Error
#include <iostream>
#include <type_traits>

using namespace std;

// USing template to avoid errors
template <typename T>
void print_type(T t)
{
    // For integer
    if (is_integral<T>::value) {
        cout << "T is an integral type" << endl;
    }

    // For floating number
    else if (is_floating_point<T>::value) {
        cout << "T is a floating point type" << endl;
    }

    // All other
    else {
        cout << "T is not an integral"
             << "or floating point type" << endl;
    }
}

// Driver Code
int main()
{
    // T is an integral type
    print_type(10);

    // T is a floating point type
    print_type(10.5f);

    // T is an integral type
    print_type(true);

    // T is an integral type
    print_type('a');

    // T is not an integral
    //  or floating point type
    print_type("GFG");

    return 0;
}
Output

T is an integral type
T is a floating point type
T is an integral type
T is an integral type
T is not an integralor floating point type

In this example, the print_type() function is a template function that takes a single argument of type T. Inside the function, there are if statements that use the is_integral and is_floating_point type traits to check the type of T. If T is of the expected type, the corresponding branch of the if statement is executed and the appropriate message is printed. If T is not of the expected type, the else branch is executed and the message “T is not an integral or floating point type” is printed.

SFINAE is used in this example because the if statement uses type traits to check the type of T, and if T is not of the expected type, the template substitution will fail. However, because SFINAE is in effect, this failure is not treated as a compile-time error, and the else branch is executed instead.
Disadvantages of SFINAE

Although SFINAE is very useful it comes with its own limitations, these are :

    It can be difficult to debug and understand the underlying code, as it relies heavily on the templates and multiple overloads to work correctly.
    SFINAE is not widely used and can be difficult to find documentation and tutorials for.
    SFINAE can produce unexpected behavior, as it relies on the compiler to determine which template to use based on the given types. This can lead to unexpected results if the compiler chooses the wrong template.
    SFINAE can also be slow, as it involves multiple checks in order to determine the correct template. This can lead to a performance hit if used in a critical section of code.

There are other methods to perform the same task one of which is enable_if
Enable_if (Syntax)

template<typename T>
enable_if_t<is_integral<T>, void> f(T x)
{
  // code that only applies to integral tmodule jinvertertb;
  reg a;
  wire y;

  //Design Instance
  jinverter jinv(y,a);

	initial
	begin
		$display ("RESULT\ta\ty");

		a = 1; # 100; // Another value
		if ( y == 0 ) // Test for inversion
			$display ("  PASS  \t%d\t%d",a,y);
		else
			$display ("  FAIL \t%d\t%d",a,y);

		a = 0; # 100; // Initial value is set
		if ( y == 1 ) // Test for inversion
			$display ("  PASS  \t%d\t%d",a,y);
		else
			$display ("  FAIL  \t%d\t%d",a,y);

		a = 1; # 50; // Another value
		if ( y == 0 ) // Test for inversion
			$display ("  PASS  \t%d\t%d",a,y);
		else
			$display ("  FAIL  \t%d\t%d",a,y);

		a = 0; # 100; // Initial value is set
		if ( y == 1 ) // Test for inversion
			$display ("  PASS  \t%d\t%d",a,y);
		else
			$display ("  FAIL  \t%d\t%d",a,y);

	end

  //enabling the wave dump
  initial begin
    $dumpfile("dump.vcd"); $dumpvars;
  end
endmoduleypes goes here
}


module jinvertertb;
  reg a;
  wire y;

  //Design Instance
  jinverter jinv(y,a);

	initial
	begin
		$display ("RESULT\ta\ty");

		a = 1; # 100; // Another value
		if ( y == 0 ) // Test for inversion
			$display ("  PASS  \t%d\t%d",a,y);
		else
			$display ("  FAIL \t%d\t%d",a,y);

		a = 0; # 100; // Initial value is set
		if ( y == 1 ) // Test for inversion
			$display ("  PASS  \t%d\t%d",a,y);
		else
			$display ("  FAIL  \t%d\t%d",a,y);

		a = 1; # 50; // Another value
		if ( y == 0 ) // Test for inversion
			$display ("  PASS  \t%d\t%d",a,y);
		else
			$display ("  FAIL  \t%d\t%d",a,y);

		a = 0; # 100; // Initial value is set
		if ( y == 1 ) // Test for inversion
			$display ("  PASS  \t%d\t%d",a,y);
		else
			$display ("  FAIL  \t%d\t%d",a,y);

	end

  //enabling the wave dump
  initial begin
    $dumpfile("dump.vcd"); $dumpvars;
  end
endmodule


module jFIFOTb;
  wire [7:0] DATAOUT;
  wire full, empty;
  reg clock, reset, wn, rn;
  reg [7:0] DATAIN;

  jFIFO jfifo(DATAOUT, full, empty, clock, reset, wn, rn, DATAIN);


  //enabling the wave dump
  initial begin
    $dumpfile("dump.vcd"); $dumpvars;
  end

  initial
  begin
    clock = 0; DATAIN = 8'd0;
    reset = 1; clock = 1; #5 ; clock = 0; #5;
    reset = 0;

    $display("Start testing");

    // First write some data into the queue
    wn = 1; rn = 0;
    DATAIN = 8'd100;
    clock = 1; #5 ; clock = 0; #5;
    DATAIN = 8'd150;
    clock = 1; #5 ; clock = 0; #5;
    DATAIN = 8'd200;
    clock = 1; #5 ; clock = 0; #5;
    DATAIN = 8'd40;
    clock = 1; #5 ; clock = 0; #5;
    DATAIN = 8'd70;
    clock = 1; #5 ; clock = 0; #5;
    DATAIN = 8'd65;
    clock = 1; #5 ; clock = 0; #5;
    DATAIN = 8'd15;
    clock = 1; #5 ; clock = 0; #5;

    // Now start reading and checking the values
    wn = 0; rn = 1;
    clock = 1; #5 ; clock = 0; #5;
    if ( DATAOUT === 8'd100 )
      $display("PASS %p ", DATAOUT);
    else
      $display("FAIL %p ", DATAOUT);

    clock = 1; #5 ; clock = 0; #5;
    if ( DATAOUT === 8'd150 )
      $display("PASS %p ", DATAOUT);
    else
      $display("FAIL %p ", DATAOUT);

    clock = 1; #5 ; clock = 0; #5;
    if ( DATAOUT === 8'd200 )
      $display("PASS %p ", DATAOUT);
    else
      $display("FAIL %p ", DATAOUT);

    clock = 1; #5 ; clock = 0; #5;
    if ( DATAOUT === 8'd40 )
      $display("PASS %p ", DATAOUT);
    else
      $display("FAIL %p ", DATAOUT);

    clock = 1; #5 ; clock = 0; #5;
    if ( DATAOUT === 8'd70 )
      $display("PASS %p ", DATAOUT);
    else
      $display("FAIL %p ", DATAOUT);

    clock = 1; #5 ; clock = 0; #5;
    if ( DATAOUT === 8'd65 )
      $display("PASS %p ", DATAOUT);
    else
      $display("FAIL %p ", DATAOUT);

    clock = 1; #5 ; clock = 0; #5;
    if ( DATAOUT === 8'd15 )
      $display("PASS %p ", DATAOUT);
    else
      $display("FAIL %p ", DATAOUT);

    clock = 1; #5 ; clock = 0; #5;
    if ( empty === 1 )
      $display("PASS %p ", empty);
    else
      $display("FAIL %p ", empty);
  end

endmodule


To understand this example, you should have the knowledge of the following Java programming topics:

    Java for Loop
    Java while and do...while Loop

The factorial of a positive number n is given by:

factorial of n (n!) = 1 * 2 * 3 * 4 * ... * n

Example 1: Find Factorial of a number using for loop

public class Factorial {

    public static void main(String[] args) {

        int num = 10;
        long factorial = 1;
        for(int i = 1; i <= num; ++i)
        {
            // factorial = factorial * i;
            factorial *= i;
        }
        System.out.printf("Factorial of %d = %d", num, factorial);
    }
}

Output

Factorial of 10 = 3628800

In this program, we've used for loop to loop through all numbers between 1 and the given number num (10), and the product of each number till num is stored in a variable factorial.

We've used long instead of int to store large results of factorial. However, it's still not big enough to store the value of bigger numbers (say 100).

For results that cannot be stored in a long variable, we use BigInteger variable declared in java.math library.
Example 2: Find Factorial of a number using BigInteger

import java.math.BigInteger;

public class Factorial {

    public static void main(String[] args) {

        int num = 30;
        BigInteger factorial = BigInteger.ONE;
        for(int i = 1; i <= num; ++i)
        {
            // factorial = factorial * i;
            factorial = factorial.multiply(BigInteger.valueOf(i));
        }
        System.out.printf("Factorial of %d = %d", num, factorial);
    }
}

Output

Factorial of 30 = 265252859812191058636308480000000

Here, instead of long, we use BigInteger variable factorial.

Since, * cannot be used with BigInteger, we instead use multiply() for the product. Also, num should be casted to BigInteger for multiplication.

Likewise, we can also use a while loop to solve this problem.
Example 3: Find Factorial of a number using while loop

public class Factorial {

    public static void main(String[] args) {

        int num = 5, i = 1;
        long factorial = 1;
        while(i <= num)
        {
            factorial *= i;
            i++;
        }
        System.out.printf("Factorial of %d = %d", num, factorial);
    }
}

Output

Factorial of 5 = 120

In the above program, unlike a for loop, we have to increment the value of i inside the body of the loop.

Though both programs are technically correct, it is better to use for loop in this case. It's because the number of iteration (upto num) is known.

Visit this page to learn to find factorial of a number using recursion.



Java Program to Check Whether a Number can be Expressed as Sum of Two Prime Numbers

To understand this example, you should have the knowledge of the following Java programming topics:

    Java Methods
    Java for Loop
    Java if...else Statement

Example: Represent a number as Sum of Two Prime Numbers

public class Main {

  public static void main(String[] args) {
    int number = 34;
    boolean flag = false;
    for (int i = 2; i <= number / 2; ++i) {

      // condition for i to be a prime number
      if (checkPrime(i)) {

        // condition for n-i to be a prime number
        if (checkPrime(number - i)) {

          // n = primeNumber1 + primeNumber2
          System.out.printf("%d = %d + %d\n", number, i, number - i);
          flag = true;
        }

      }
    }

    if (!flag)
      System.out.println(number + " cannot be expressed as the sum of two prime numbers.");
  }

  // Function to check prime number
  static boolean checkPrime(int num) {
    boolean isPrime = true;

    for (int i = 2; i <= num / 2; ++i) {
      if (num % i == 0) {
        isPrime = false;
        break;
      }
    }

    return isPrime;
  }
}

Run Code

Output

34 = 3 + 31
34 = 5 + 29
34 = 11 + 23
34 = 17 + 17

In the above example, we have created the checkPrime() method to find whether a number is prime or not. The method returns true if the passed number is prime.

Here, we have a number 34. The program tries to check if 34 can be represented as the sum of two prime numbers.

Working of Program

    First, we run a for loop from i = 2 to number / 2.
    Inside the for loop, we used two if statements. The first statement checks if i is prime or not.

    If true, the second if statement checks if number - i is prime or not. This is because the sum of i and number - i is equal to number.
    If the second statement is also true, then we can say the number 34 is a valid sum of two prime numbers.




// Java Program to Check if Given Integer is Odd or Even
// Using Brute Forcew Approach

// Importing required classes
import java.io.*;
import java.util.Scanner;

// Main class
class GFG {

    // Main Driver Method
    public static void main(String[] args)
    {
        // Declaring and initializing integer variable
        int num = 10;

        // Checking if number is even or odd number
        // via remainder
        if (num % 2 == 0) {

            // If remainder is zero then this number is even
            System.out.println("Entered Number is Even");
        }

        else {

            // If remainder is not zero then this number is
            // odd
            System.out.println("Entered Number is Odd");
        }
    }
}




// Java Program to find even sum of
// fibonacci Series Till number N
import java.io.*;

class geeksforgeeks {

    // Computing the value of first fibonacci series
    // and storing the sum of even indexed numbers
    static int Fib_Even_Sum(int N)
    {
        if (N <= 0)
            return 0;

        int fib[] = new int[2 * N + 1];
        fib[0] = 0;
        fib[1] = 1;

        // Initializing the sum
        int s = 0;

        // Adding remaining numbers
        for (int j = 2; j <= 2 * N; j++) {
            fib[j] = fib[j - 1] + fib[j - 2];

            // Only considering even indexes
            if (j % 2 == 0)
                s += fib[j];
        }

        return s;
    }

    // The Driver code
    public static void main(String[] args)
    {
        int N = 11;

        // Prints the sum of even-indexed numbers
        System.out.println(
            "Even sum of fibonacci series till number " + N
            + " is: " + +Fib_Even_Sum(N));
    }
}



    It can be clearly seen that the required sum can be obtained thus:
    2 ( F2 + F4 + F6 +………+ F2n ) = (F1 + F2 + F3 + F4 +………+ F2n) – (F1 – F2 + F3 – F4 +………+ F2n)

    Now the first term can be obtained if we put 2n instead of n in the formula given here.

    Thus F1 + F2 + F3 + F4 +………+ F2n = F2n+2 – 1.

    The second term can also be found if we put 2n instead of n in the formula given here

    Thus, F1 – F2 + F3 – F4 +………- F2n = 1 + (-1)2n+1F2n-1 = 1 – F2n-1.

    So, 2 ( F2 + F4 + F6 +………+ F2n)
    = F2n+2 – 1 – 1 + F2n-1
    = F2n+2 + F2n-1 – 2
    = F2n + F2n+1 + F2n+1 – F2n – 2
    = 2 ( F2n+1 -1)
    Hence, ( F2 + F4 + F6 +………+ F2n) = F2n+1 -1 .



// Java Program to find even indexed
// Fibonacci Sum in O(Log n) time.

class GFG {

    static int MAX = 1000;

    // Create an array for memoization
    static int f[] = new int[MAX];

    // Returns n'th Fibonacci number
    // using table f[]
    static int fib(int n)
    {
        // Base cases
        if (n == 0) {
            return 0;
        }
        if (n == 1 || n == 2) {
            return (f[n] = 1);
        }

        // If fib(n) is already computed
        if (f[n] == 1) {
            return f[n];
        }

        int k = (n % 2 == 1) ? (n + 1) / 2 : n / 2;

        // Applying above formula [Note value n&1 is 1
        // if n is odd, else 0].
        f[n] = (n % 2 == 1)
                   ? (fib(k) * fib(k)
                      + fib(k - 1) * fib(k - 1))
                   : (2 * fib(k - 1) + fib(k)) * fib(k);

        return f[n];
    }

    // Computes value of even-indexed Fibonacci Sum
    static int calculateEvenSum(int n)
    {
        return (fib(2 * n + 1) - 1);
    }

    // Driver program to test above function
    public static void main(String[] args)
    {
        // Get n
        int n = 11;

        // Find the alternating sum
        System.out.println(
            "Even indexed Fibonacci Sum upto " + n
            + " terms: " + calculateEvenSum(n));
    }
}

SCREEN 13
z = 1 / 100
FOR i = 0 TO 255
  PALETTE i, INT(i / 4) + INT(i / 4) * 256 + INT(i / 4) * 65536
NEXT
DO
  y = 1
  dy = 0
  ox = 0: oy = 1
  FOR x = 1 TO 430
    IF y > 0 THEN dy = dy - 1 / 4000
    IF y < 0 THEN dy = dy + z
    y = y + dy
    a = POINT(x, 100 + y * 50)
    a = a + 1
    IF a > 255 THEN a = 255
    PSET (x, 100 + y * 50), a
  NEXT
  z = z - 1 / 10000
LOOP


DIM bodyx(70)
DIM bodyy(70)
DIM bodyvx(70)
DIM bodyvy(70)
DIM obodyx(70)
DIM obodyy(70)

RANDOMIZE TIMER

bodies = 30
FOR i = 1 TO bodies
  bodyx(i) = 620 * RND + 10
  bodyy(i) = 400 * RND + 30
  bodyvx(i) = RND - .5
  bodyvy(i) = RND - .5
NEXT


SCREEN 12

PRINT "...."
WHILE INKEY$ = "": WEND
CLS





DO

'   apply gravity

FOR i = 1 TO bodies
  bodyvy(i) = bodyvy(i) + .001
NEXT

'  adjust distances

FOR i = 1 TO bodies
  FOR j = 1 TO bodies
    IF i <> j THEN
      centx = (bodyx(i) + bodyx(j)) * .5
      centy = (bodyy(i) + bodyy(j)) * .5

      distx = (bodyx(i) - bodyx(j)) / 2
      disty = (bodyy(i) - bodyy(j)) / 2
      distf = SQR(distx * distx + disty * disty)

      mx = distx / distf
      my = disty / distf

      bodyvx(i) = bodyvx(i) - mx * (distf - 50) / 100
      bodyvy(i) = bodyvy(i) - my * (distf - 50) / 100
    END IF
  NEXT
NEXT

'   collide with ground

FOR i = 1 TO bodies
  IF bodyy(i) + bodyvy(i) > 440 OR bodyy(i) + bodyvy(i) < 5 THEN
    bodyvy(i) = -bodyvy(i) * 0!
    bodyvx(i) = bodyvx(i) * 0!
  END IF
  IF bodyx(i) + bodyvx(i) > 625 OR bodyx(i) + bodyvx(i) < 15 THEN
    bodyvx(i) = -bodyvx(i) * 0!
    bodyvy(i) = bodyvy(i) * 0!
  END IF
NEXT

'  move bodies

FOR i = 1 TO bodies
  obodyx(i) = bodyx(i)
  obodyy(i) = bodyy(i)
  bodyx(i) = bodyx(i) + bodyvx(i)
  bodyy(i) = bodyy(i) + bodyvy(i)
NEXT

'  clear/draw

q = (q + 1) MOD 16

FOR i = 1 TO bodies
  COLOR 0
  CIRCLE (obodyx(i), obodyy(i)), 4
  COLOR 15
  CIRCLE (bodyx(i), bodyy(i)), 4
  'COLOR q
  'LINE (bodyx(i), bodyy(i))-(bodyx(i) + bodyvx(i) * 30, bodyy(i) + bodyvy(i) * 30)
NEXT

COLOR 2
LINE (obodyx(1), obodyy(1))-(bodyx(1), bodyy(1))

'FOR i = 1 TO bodies - 1
'  FOR j = i + 1 TO bodies
'    COLOR 0
'    LINE (obodyx(i), obodyy(i))-(obodyx(j), obodyy(j))
'    COLOR 2
'    LINE (bodyx(i), bodyy(i))-(bodyx(j), bodyy(j))
'  NEXT
'NEXT

LOOP



;---------------------------------------------------------------------------
;  start
;---------------------------------------------------------------------------

start
  desc
    msg "It's a room."
  objects
    start/key
    start/door

start/key
  id "key"
  desc
    msg "It's a key."
  actions
    use door
      ifn start/door.flags.locked
        msg "The door isn't locked."
      if start/door.flags.locked
        msg "The lock on the door goes 'click'."
        set start/door.flags.locked false
    examine
      msg "It is shiny."

start/door
  id "door"
  desc
    msg "You see a door."
  flags
    locked true
    open false
  actions
    open
      if start/door.flags.locked
        msg "It is locked."
      ifn start/door.flags.locked
        msg "The door opens."
    close
      ifn start/door.flags.open
        msg "It is already closed."
      if start/door.flags.open
        msg "The door closes."
    go
      if start/door.flags.open
        goto end
      ifn start/door.flags.open
        msg "The door is closed."
    examine
      msg "It looks just like a door."

;---------------------------------------------------------------------------
;  end
;---------------------------------------------------------------------------

end
  desc
    "Congratimafations! Another room."
  objects
    start/door




macroScript ExportCamera category:"Cameras" toolTip:"Blech!" Icon:#("Cameras",2)
(
	filename = maxFileName
    flen = findString filename ".max"
	filename = substring filename 1 (flen-1)

	sname = filename

    filename = maxFilePath + filename + ".piper3d"
	deletefile filename
	out_file = createfile filename



	format ";------------------------------------------------------------\n" to:out_file
	format ";   Piper 3D scene file\n" to:out_file
	format ";\n" to:out_file
	format ";   Scene: % \n" sname to:out_file
	format ";------------------------------------------------------------\n" to:out_file
	format "\n" to:out_file
	format "%_begin\n" sname to:out_file
	format "\n" to:out_file

	animend = animationRange.end as string
	animend = substring animend 1 (findString animend "f"-1)
	format "		dc.w 	%	; no. frames\n" animend to:out_file
	format "		dc.w 	%	; no. objects\n" objects.count to:out_file

	faces = 0
	vertices = 0
	for o in objects do (
		if  classOf o == Editable_Mesh then (
			faces += o.numfaces
			vertices += o.numverts
		)
	)

	format "		dc.w 	%	; no. vertices\n" vertices to:out_file
	format "		dc.w 	%	; no. faces\n" faces to:out_file
	format "\n" to:out_file

	format "		dc.w	0   ; current frame\n" to:out_file
	format "\n" to:out_file

	format "		dc.l	%_vtx  ; begin vertices\n" sname sname to:out_file
	format "		dc.l	%_fcs  ; begin fæces\n" sname sname to:out_file
	format "		dc.l	%_mtx  ; begin matrices\n" sname sname to:out_file
	format "		dc.l	0  ; begin vertex normals\n" to:out_file
	format "\n" to:out_file

	format "		dc.w	0  ; init flag\n" to:out_file
	format "\n" to:out_file

	format ";------------------------------------------------------------\n" to:out_file
	format ";   Vertex groups\n" to:out_file
	format ";------------------------------------------------------------\n" to:out_file
	format "\n" to:out_file
	format "%_vtx\n" sname to:out_file
	format "\n" to:out_file

	for i=1 to objects.count do (
		format ";   group %: %\n" (i-1) objects[i].name to:out_file
		format "\n" to:out_file

		if  classOf objects[i] == Editable_Mesh then (
			obimat = (inverse objects[i].transform)
			themesh = snapshotAsMesh objects[i]
			format "		dc.w 	%  ; no. vertices\n" themesh.numverts to:out_file
			for j=1 to themesh.numverts do (
				v = (getVert themesh j) * obimat
				vxs = formatf v.x 6
				vys = formatf v.y 6
				vzs = formatf v.z 6
				format "		dc.s 	%,%,%\n" vxs vys vzs to:out_file
			)
		) else (
			format "		dc.w 0  ; not a mesh\n" to:out_file
		)

		format "\n" to:out_file
	)

	format ";------------------------------------------------------------\n" to:out_file
	format ";   Fæces\n" to:out_file
	format ";------------------------------------------------------------\n" to:out_file
	format "\n" to:out_file
	format "%_fcs\n" sname to:out_file
	format "\n" to:out_file

	voff = 0

	for i=1 to objects.count do (
		format ";   group %: %\n" (i-1) objects[i].name to:out_file
		format "\n" to:out_file

		if  classOf objects[i] == Editable_Mesh then (
			themesh = snapshotAsMesh objects[i]
			for j=1 to themesh.numfaces do (
				f = getFace themesh j
				format "		dc.w	%,%,%\n" (int (f.x-1+voff)) (int (f.y-1+voff)) (int (f.z-1+voff)) to:out_file

				smat = objects[i].material
				mattype = "0"

				u1 = 0
				v1 = 0
				u2 = 0
				v2 = 0
				u3 = 0
				v3 = 0

				if smat == undefined then (
					matname = "0"
				) else (
					if getNumSubMtls smat == 0 then (
						mat = smat
					) else (
						matid = getFaceMatID themesh j
					 	mat = smat.MaterialList[matid]
					)
					matname = (int mat.diffuse.r)*256*256 + (int mat.diffuse.g)*256 + (int mat.diffuse.b)
					dmap = mat.diffuseMap
					if dmap != undefined then (

						mattype = "1"
						matname = dmap.filename
						fntokens = filterString matname "\\"
						matname = fntokens[fntokens.count]
						fntokens = filterString matname "."
						matname = fntokens[1]

						fntokens = filterString matname "-"
						matname = ""
						for k=1 to fntokens.count do (
							matname = matname + fntokens[k]
							if k < fntokens.count then matname=matname+"m"
						)
						matname = sname + "_" + matname + "+1024+14"

						tverts = getTVFace theMesh j
						u1 = (int (getTVert theMesh tverts[1] * 65535).x)
						v1 = 65535 - (int (getTVert theMesh tverts[1] * 65535).y)
						u2 = (int (getTVert theMesh tverts[2] * 65535).x)
						v2 = 65536 - (int (getTVert theMesh tverts[2] * 65535).y)

macroScript ExportCamera category:"Cameras" toolTip:"Blech!" Icon:#("Cameras",2)
(
	filename = maxFileName
    flen = findString filename ".max"
	filename = substring filename 1 (flen-1)

	sname = filename

    filename = maxFilePath + filename + ".piper3d"
	deletefile filename
	out_file = createfile filename



	format ";------------------------------------------------------------\n" to:out_file
	format ";   Piper 3D scene file\n" to:out_file
	format ";\n" to:out_file
	format ";   Scene: % \n" sname to:out_file
	format ";------------------------------------------------------------\n" to:out_file
	format "\n" to:out_file
	format "%_begin\n" sname to:out_file
	format "\n" to:out_file

	animend = animationRange.end as string
	animend = substring animend 1 (findString animend "f"-1)
	format "		dc.w 	%	; no. frames\n" animend to:out_file
	format "		dc.w 	%	; no. objects\n" objects.count to:out_file

	faces = 0
	vertices = 0
	for o in objects do (
		if  classOf o == Editable_Mesh then (
			faces += o.numfaces
			vertices += o.numverts
		)
	)

	format "		dc.w 	%	; no. vertices\n" vertices to:out_file
	format "		dc.w 	%	; no. faces\n" faces to:out_file
	format "\n" to:out_file

	format "		dc.w	0   ; current frame\n" to:out_file
	format "\n" to:out_file

	format "		dc.l	%_vtx  ; begin vertices\n" sname sname to:out_file
	format "		dc.l	%_fcs  ; begin fæces\n" sname sname to:out_file
	format "		dc.l	%_mtx  ; begin matrices\n" sname sname to:out_file
	format "		dc.l	0  ; begin vertex normals\n" to:out_file
	format "\n" to:out_file

	format "		dc.w	0  ; init flag\n" to:out_file
	format "\n" to:out_file

	format ";------------------------------------------------------------\n" to:out_file
	format ";   Vertex groups\n" to:out_file
	format ";------------------------------------------------------------\n" to:out_file
	format "\n" to:out_file
	format "%_vtx\n" sname to:out_file
	format "\n" to:out_file

	for i=1 to objects.count do (
		format ";   group %: %\n" (i-1) objects[i].name to:out_file
		format "\n" to:out_file

		if  classOf objects[i] == Editable_Mesh then (
			obimat = (inverse objects[i].transform)
			themesh = snapshotAsMesh objects[i]
			format "		dc.w 	%  ; no. vertices\n" themesh.numverts to:out_file
			for j=1 to themesh.numverts do (
				v = (getVert themesh j) * obimat
				vxs = formatf v.x 6
				vys = formatf v.y 6
				vzs = formatf v.z 6
				format "		dc.s 	%,%,%\n" vxs vys vzs to:out_file
			)
		) else (
			format "		dc.w 0  ; not a mesh\n" to:out_file
		)

		format "\n" to:out_file
	)

	format ";------------------------------------------------------------\n" to:out_file
	format ";   Fæces\n" to:out_file
	format ";------------------------------------------------------------\n" to:out_file
	format "\n" to:out_file
	format "%_fcs\n" sname to:out_file
	format "\n" to:out_file

	voff = 0

	for i=1 to objects.count do (
		format ";   group %: %\n" (i-1) objects[i].name to:out_file
		format "\n" to:out_file

		if  classOf objects[i] == Editable_Mesh then (
			themesh = snapshotAsMesh objects[i]
			for j=1 to themesh.numfaces do (
				f = getFace themesh j
				format "		dc.w	%,%,%\n" (int (f.x-1+voff)) (int (f.y-1+voff)) (int (f.z-1+voff)) to:out_file

				smat = objects[i].material
				mattype = "0"

				u1 = 0
				v1 = 0
				u2 = 0
				v2 = 0
				u3 = 0
				v3 = 0

				if smat == undefined then (
					matname = "0"
				) else (
					if getNumSubMtls smat == 0 then (
						mat = smat
					) else (
						matid = getFaceMatID themesh j
					 	mat = smat.MaterialList[matid]
					)
					matname = (int mat.diffuse.r)*256*256 + (int mat.diffuse.g)*256 + (int mat.diffuse.b)
					dmap = mat.diffuseMap
					if dmap != undefined then (

						mattype = "1"
						matname = dmap.filename
						fntokens = filterString matname "\\"
						matname = fntokens[fntokens.count]
						fntokens = filterString matname "."
						matname = fntokens[1]

						fntokens = filterString matname "-"
						matname = ""
						for k=1 to fntokens.count do (
							matname = matname + fntokens[k]
							if k < fntokens.count then matname=matname+"m"
						)
						matname = sname + "_" + matname + "+1024+14"

						tverts = getTVFace theMesh j
						u1 = (int (getTVert theMesh tverts[1] * 65535).x)
						v1 = 65535 - (int (getTVert theMesh tverts[1] * 65535).y)
						u2 = (int (getTVert theMesh tverts[2] * 65535).x)
						v2 = 65536 - (int (getTVert theMesh tverts[2] * 65535).y)
						u3 = (int (getTVert theMesh tverts[3] * 65535).x)
						v3 = 65536 - (int (getTVert theMesh tverts[3] * 65535).y)

						if u1 > 65535 then u1 = 65535
						if v1 > 65535 then v1 = 65535
						if u2 > 65535 then u2 = 65535
						if v2 > 65535 then v2 = 65535
						if u3 > 65535 then u3 = 65535
						if v3 > 65535 then v3 = 65535

						/*
						if u1 < 0 then u1 = 0
						if v1 < 0 then v1 = 0
						if u2 < 0 then u2 = 0
						if v2 < 0 then v2 = 0
						if u3 < 0 then u3 = 0
						if v3 < 0 then v3 = 0
						*/

						if mat.opacityMap != undefined then (
							mattype = "2"
						)
					)
				)

				format "		dc.w	%\n" mattype to:out_file
				format "		dc.l 	%\n" matname to:out_file
				format "		dc.w 	%,%,%,%,%,%\n" u1 v1 u2 v2 u3 v3 to:out_file

			)
			format "\n" to:out_file

			voff += themesh.numverts
		)
	)

	format ";------------------------------------------------------------\n" to:out_file
	format ";   \n" to:out_file
	format ";------------------------------------------------------------\n" to:out_file
	format "\n" to:out_file
	format "%_mtx\n" sname to:out_file
	format "\n" to:out_file

	for i = animationRange.start to animationRange.end do (

	    sliderTime=i

		format ";   index %\n" i to:out_file
		format "\n" to:out_file

		for j = 1 to objects.count do (

			o=objects[j]

			vpm=getViewTM()
			tm=o.transform*vpm
			tm1=tm.row1
			tm2=tm.row2
			tm3=tm.row3
			tm4=tm.row4
			tm11=formatf tm1.x 6
			tm12=formatf tm1.y 6
			tm13=formatf tm1.z 6
			tm21=formatf tm2.x 6
			tm22=formatf tm2.y 6
			tm23=formatf tm2.z 6
			tm31=formatf tm3.x 6
			tm32=formatf tm3.y 6
			tm33=formatf tm3.z 6
			tm41=formatf tm4.x 6
			tm42=formatf tm4.y 6
			tm43=formatf tm4.z 6
			format "		dc.s	%,%,%\n" tm11 tm12 tm13 to:out_file
			format "		dc.s	%,%,%\n" tm21 tm22 tm23 to:out_file
			format "		dc.s	%,%,%\n" tm31 tm32 tm33 to:out_file
			format "		dc.s	%,%,%\n" tm41 tm42 tm43 to:out_file
			format "\n" to:out_file
		)

	)

	close out_file
	shelllaunch "notepad.exe" filename

)

-- theend	u3 = (int (getTVert theMesh tverts[3] * 65535).x)
						v3 = 65536 - (int (getTVert theMesh tverts[3] * 65535).y)

						if u1 > 65535 then u1 = 65535
						if v1 > 65535 then v1 = 65535
						if u2 > 65535 then u2 = 65535
						if v2 > 65535 then v2 = 65535
						if u3 > 65535 then u3 = 65535
						if v3 > 65535 then v3 = 65535

						/*
						if u1 < 0 then u1 = 0
						if v1 < 0 then v1 = 0
						if u2 < 0 then u2 = 0
						if v2 < 0 then v2 = 0
						if u3 < 0 then u3 = 0
						if v3 < 0 then v3 = 0
						*/

						if mat.opacityMap != undefined then (
							mattype = "2"
						)
					)
				)

				format "		dc.w	%\n" mattype to:out_file
				format "		dc.l 	%\n" matname to:out_file
				format "		dc.w 	%,%,%,%,%,%\n" u1 v1 u2 v2 u3 v3 to:out_file

			)
			format "\n" to:out_file

			voff += themesh.numverts
		)
	)

	format ";------------------------------------------------------------\n" to:out_file
	format ";   \n" to:out_file
	format ";------------------------------------------------------------\n" to:out_file
	format "\n" to:out_file
	format "%_mtx\n" sname to:out_file
	format "\n" to:out_file

	for i = animationRange.start to animationRange.end do (

	    sliderTime=i

		format ";   index %\n" i to:out_file
		format "\n" to:out_file

		for j = 1 to objects.count do (

			o=objects[j]

			vpm=getViewTM()
			tm=o.transform*vpm
			tm1=tm.row1
			tm2=tm.row2
			tm3=tm.row3
			tm4=tm.row4
			tm11=formatf tm1.x 6
			tm12=formatf tm1.y 6
			tm13=formatf tm1.z 6
			tm21=formatf tm2.x 6
			tm22=formatf tm2.y 6
			tm23=formatf tm2.z 6
			tm31=formatf tm3.x 6
			tm32=formatf tm3.y 6
			tm33=formatf tm3.z 6
			tm41=formatf tm4.x 6
			tm42=formatf tm4.y 6
			tm43=formatf tm4.z 6
			format "		dc.s	%,%,%\n" tm11 tm12 tm13 to:out_file
			format "		dc.s	%,%,%\n" tm21 tm22 tm23 to:out_file
			format "		dc.s	%,%,%\n" tm31 tm32 tm33 to:out_file
			format "		dc.s	%,%,%\n" tm41 tm42 tm43 to:out_file
			format "\n" to:out_file
		)

	)

	close out_file
	shelllaunch "notepad.exe" filename

)

-- theend


void DS3231::setDateTime(uint16_t year, uint8_t month, uint8_t day, uint8_t hour, uint8_t minute, uint8_t second)
{
    Wire.beginTransmission(DS3231_ADDRESS);

    #if ARDUINO >= 100
        Wire.write(DS3231_REG_TIME);
    #else
        Wire.send(DS3231_REG_TIME);
    #endif

    #if ARDUINO >= 100
        Wire.write(dec2bcd(second));
        Wire.write(dec2bcd(minute));
        Wire.write(dec2bcd(hour));
        Wire.write(dec2bcd(dow(year, month, day)));
        Wire.write(dec2bcd(day));
        Wire.write(dec2bcd(month));
        Wire.write(dec2bcd(year-2000));
    #else
        Wire.send(dec2bcd(second));
        Wire.send(dec2bcd(minute));
        Wire.send(dec2bcd(hour));
        Wire.send(dec2bcd(dow(year, month, day)));
        Wire.send(dec2bcd(day));
        Wire.send(dec2bcd(month));
        Wire.send(dec2bcd(year-2000));
    #endif

    #if ARDUINO >= 100
        Wire.write(DS3231_REG_TIME);
    #else
        Wire.send(DS3231_REG_TIME);
    #endif

    Wire.endTransmission();
}

void DS3231::setDateTime(uint32_t t)
{
    t -= 946681200;

    uint16_t year;
    uint8_t month;
    uint8_t day;
    uint8_t hour;
    uint8_t minute;
    uint8_t second;

    second = t % 60;
    t /= 60;

    minute = t % 60;
    t /= 60;

    hour = t % 24;
    uint16_t days = t / 24;
    uint8_t leap;

    for (year = 0; ; ++year)
    {
        leap = year % 4 == 0;
        if (days < 365 + leap)
        {
            break;
        }
        days -= 365 + leap;
    }

    for (month = 1; ; ++month)
    {
        uint8_t daysPerMonth = pgm_read_byte(daysArray + month - 1);

        if (leap && month == 2)
        {
            ++daysPerMonth;
        }

        if (days < daysPerMonth)
        {
            break;
        }
        days -= daysPerMonth;
    }

    day = days + 1;

    setDateTime(year+2000, month, day, hour, minute, second);
}

void DS3231::setDateTime(const char* date, const char* time)
{
    uint16_t year;
    uint8_t month;
    uint8_t day;
    uint8_t hour;
    uint8_t minute;
    uint8_t second;

    year = conv2d(date + 9);

    switch (date[0])
    {
        case 'J': month = date[1] == 'a' ? 1 : month = date[2] == 'n' ? 6 : 7; break;
        case 'F': month = 2; break;
        case 'A': month = date[2] == 'r' ? 4 : 8; break;
        case 'M': month = date[2] == 'r' ? 3 : 5; break;
        case 'S': month = 9; break;
        case 'O': month = 10; break;
        case 'N': month = 11; break;
        case 'D': month = 12; break;
    }

    day = conv2d(date + 4);
    hour = conv2d(time);
    minute = conv2d(time + 3);
    second = conv2d(time + 6);

    setDateTime(year+2000, month, day, hour, minute, second);
}

char* DS3231::dateFormat(const char* dateFormat, RTCDateTime dt)
{
    char buffer[255];

    buffer[0] = 0;

    char helper[11];

    while (*dateFormat != '\0')
    {
        switch (dateFormat[0])
        {
            // Day decoder
            case 'd':
                sprintf(helper, "%02d", dt.day);
                strcat(buffer, (const char *)helper);
                break;
            case 'j':
                sprintf(helper, "%d", dt.day);
                strcat(buffer, (const char *)helper);
                break;
            case 'l':
                strcat(buffer, (const char *)strDayOfWeek(dt.dayOfWeek));
                break;
            case 'D':
                strncat(buffer, strDayOfWeek(dt.dayOfWeek), 3);
                break;
            case 'N':
                sprintf(helper, "%d", dt.dayOfWeek);
                strcat(buffer, (const char *)helper);
                break;
            case 'w':
                sprintf(helper, "%d", (dt.dayOfWeek + 7) % 7);
                strcat(buffer, (const char *)helper);
                break;
            case 'z':
                sprintf(helper, "%d", dayInYear(dt.year, dt.month, dt.day)+1);
                strcat(buffer, (const char *)helper);
                break;
            case 'S':
                strcat(buffer, (const char *)strDaySufix(dt.day));
                break;

            // Month decoder
            case 'm':
                sprintf(helper, "%02d", dt.month);
                strcat(buffer, (const char *)helper);
                break;
            case 'n':
                sprintf(helper, "%d", dt.month);
                strcat(buffer, (const char *)helper);
                break;
            case 'F':
                strcat(buffer, (const char *)strMonth(dt.month));
                break;
            case 'M':
                strncat(buffer, (const char *)strMonth(dt.month), 3);
                break;
            case 't':
                sprintf(helper, "%d", daysInMonth(dt.year, dt.month));
                strcat(buffer, (const char *)helper);
                break;

            // Year decoder
            case 'Y':
                sprintf(helper, "%d", dt.year);
                strcat(buffer, (const char *)helper);
                break;
            case 'y': sprintf(helper, "%02d", dt.year-2000);
                strcat(buffer, (const char *)helper);
                break;
            case 'L':
                sprintf(helper, "%d", isLeapYear(dt.year));
                strcat(buffer, (const char *)helper);
                break;

            // Hour decoder
            case 'H':
                sprintf(helper, "%02d", dt.hour);
                strcat(buffer, (const char *)helper);
                break;
            case 'G':
                sprintf(helper, "%d", dt.hour);
                strcat(buffer, (const char *)helper);
                break;
            case 'h':
                sprintf(helper, "%02d", hour12(dt.hour));
                strcat(buffer, (const char *)helper);
                break;
            case 'g':
                sprintf(helper, "%d", hour12(dt.hour));
                strcat(buffer, (const char *)helper);
                break;
            case 'A':
                strcat(buffer, (const char *)strAmPm(dt.hour, true));
                break;
            case 'a':
                strcat(buffer, (const char *)strAmPm(dt.hour, false));
                break;

            // Minute decoder
            case 'i':
                sprintf(helper, "%02d", dt.minute);
                strcat(buffer, (const char *)helper);
                break;

            // Second decoder
            case 's':
                sprintf(helper, "%02d", dt.second);
                strcat(buffer, (const char *)helper);
                break;

            // Misc decoder
            case 'U':
                sprintf(helper, "%lu", dt.unixtime);
                strcat(buffer, (const char *)helper);
                break;

            default:
                strncat(buffer, dateFormat, 1);
                break;
        }
        dateFormat++;
    }

    return buffer;
}

char* DS3231::dateFormat(const char* dateFormat, RTCAlarmTime dt)
{
    char buffer[255];

    buffer[0] = 0;

    char helper[11];

    while (*dateFormat != '\0')
    {
        switch (dateFormat[0])
        {
            // Day decoder
            case 'd':
                sprintf(helper, "%02d", dt.day);
                strcat(buffer, (const char *)helper);
                break;
            case 'j':
                sprintf(helper, "%d", dt.day);
                strcat(buffer, (const char *)helper);
                break;
            case 'l':
                strcat(buffer, (const char *)strDayOfWeek(dt.day));
                break;
            case 'D':
                strncat(buffer, strDayOfWeek(dt.day), 3);
                break;
            case 'N':
                sprintf(helper, "%d", dt.day);
                strcat(buffer, (const char *)helper);
                break;
            case 'w':
                sprintf(helper, "%d", (dt.day + 7) % 7);
                strcat(buffer, (const char *)helper);
                break;
            case 'S':
                strcat(buffer, (const char *)strDaySufix(dt.day));
                break;

            // Hour decoder
            case 'H':
                sprintf(helper, "%02d", dt.hour);
                strcat(buffer, (const char *)helper);
                break;
            case 'G':
                sprintf(helper, "%d", dt.hour);
                strcat(buffer, (const char *)helper);
                break;
            case 'h':
                sprintf(helper, "%02d", hour12(dt.hour));
                strcat(buffer, (const char *)helper);
                break;
            case 'g':
                sprintf(helper, "%d", hour12(dt.hour));
                strcat(buffer, (const char *)helper);
                break;
            case 'A':
                strcat(buffer, (const char *)strAmPm(dt.hour, true));
                break;
            case 'a':
                strcat(buffer, (const char *)strAmPm(dt.hour, false));
                break;

            // Minute decoder
            case 'i':
                sprintf(helper, "%02d", dt.minute);
                strcat(buffer, (const char *)helper);
                break;

            // Second decoder
            case 's':
                sprintf(helper, "%02d", dt.second);
                strcat(buffer, (const char *)helper);
                break;

            default:
                strncat(buffer, dateFormat, 1);
                break;
        }
        dateFormat++;
    }

    return buffer;
}

RTCDateTime DS3231::getDateTime(void)
{
    int values[7];

    Wire.beginTransmission(DS3231_ADDRESS);
    #if ARDUINO >= 100
        Wire.write(DS3231_REG_TIME);
    #else
        Wire.send(DS3231_REG_TIME);
    #endif
    Wire.endTransmission();

    Wire.requestFrom(DS3231_ADDRESS, 7);

    while(!Wire.available()) {};

    for (int i = 6; i >= 0; i--)
    {
        #if ARDUINO >= 100
            values[i] = bcd2dec(Wire.read());
        #else
            values[i] = bcd2dec(Wire.receive());
        #endif
    }

    Wire.endTransmission();

    t.year = values[0] + 2000;
    t.month = values[1];
    t.day = values[2];
    t.dayOfWeek = values[3];
    t.hour = values[4];
    t.minute = values[5];
    t.second = values[6];
    t.unixtime = unixtime();

    return t;
}

uint8_t DS3231::isReady(void)
{
    return true;
}

void DS3231::enableOutput(bool enabled)
{
    uint8_t value;

    value = readRegister8(DS3231_REG_CONTROL);

    value &= 0b11111011;
    value |= (!enabled << 2);

    writeRegister8(DS3231_REG_CONTROL, value);
}

void DS3231::setBattery(bool timeBattery, bool squareBattery)
{
    uint8_t value;

    value = readRegister8(DS3231_REG_CONTROL);

    if (squareBattery)
    {
        value |= 0b01000000;
    } else
    {
        value &= 0b10111111;
    }

    if (timeBattery)
    {
        value &= 0b01111011;
    } else
    {
        value |= 0b10000000;
    }

    writeRegister8(DS3231_REG_CONTROL, value);
}

bool DS3231::isOutput(void)
{
    uint8_t value;

    value = readRegister8(DS3231_REG_CONTROL);

    value &= 0b00000100;
    value >>= 2;

    return !value;
}

void DS3231::setOutput(DS3231_sqw_t mode)
{
    uint8_t value;

    value = readRegister8(DS3231_REG_CONTROL);

    value &= 0b11100111;
    value |= (mode << 3);

    writeRegister8(DS3231_REG_CONTROL, value);
}

DS3231_sqw_t DS3231::getOutput(void)
{
    uint8_t value;

    value = readRegister8(DS3231_REG_CONTROL);

    value &= 0b00011000;
    value >>= 3;

    return (DS3231_sqw_t)value;
}

void DS3231::enable32kHz(bool enabled)
{
    uint8_t value;

    value = readRegister8(DS3231_REG_STATUS);

    value &= 0b11110111;
    value |= (enabled << 3);

    writeRegister8(DS3231_REG_STATUS, value);
}

bool DS3231::is32kHz(void)
{
    uint8_t value;

    value = readRegister8(DS3231_REG_STATUS);

    value &= 0b00001000;
    value >>= 3;

    return value;
}


#ifndef DS3231_h
#define DS3231_h

#if ARDUINO >= 100
#include "Arduino.h"
#else
#include "WProgram.h"
#endif

#define DS3231_ADDRESS              (0x68)

#define DS3231_REG_TIME             (0x00)
#define DS3231_REG_ALARM_1          (0x07)
#define DS3231_REG_ALARM_2          (0x0B)
#define DS3231_REG_CONTROL          (0x0E)
#define DS3231_REG_STATUS           (0x0F)
#define DS3231_REG_TEMPERATURE      (0x11)

#ifndef RTCDATETIME_STRUCT_H
#define RTCDATETIME_STRUCT_H
struct RTCDateTime
{
    uint16_t year;
    uint8_t month;
    uint8_t day;
    uint8_t hour;
    uint8_t minute;
    uint8_t second;
    uint8_t dayOfWeek;
    uint32_t unixtime;
};

struct RTCAlarmTime
{
    uint8_t day;
    uint8_t hour;
    uint8_t minute;
    uint8_t second;
};
#endif

typedef enum
{
    DS3231_1HZ          = 0x00,
    DS3231_4096HZ       = 0x01,
    DS3231_8192HZ       = 0x02,
    DS3231_32768HZ      = 0x03
} DS3231_sqw_t;

typedef enum
{
    DS3231_EVERY_SECOND   = 0b00001111,
    DS3231_MATCH_S        = 0b00001110,
    DS3231_MATCH_M_S      = 0b00001100,
    DS3231_MATCH_H_M_S    = 0b00001000,
    DS3231_MATCH_DT_H_M_S = 0b00000000,
    DS3231_MATCH_DY_H_M_S = 0b00010000
} DS3231_alarm1_t;

typedef enum
{
    DS3231_EVERY_MINUTE   = 0b00001110,
    DS3231_MATCH_M        = 0b00001100,
    DS3231_MATCH_H_M      = 0b00001000,
    DS3231_MATCH_DT_H_M   = 0b00000000,
    DS3231_MATCH_DY_H_M   = 0b00010000
} DS3231_alarm2_t;

class DS3231
{
    public:

	bool begin(void);

	void setDateTime(uint16_t year, uint8_t month, uint8_t day, uint8_t hour, uint8_t minute, uint8_t second);
	void setDateTime(uint32_t t);
	void setDateTime(const char* date, const char* time);
	RTCDateTime getDateTime(void);
	uint8_t isReady(void);

	DS3231_sqw_t getOutput(void);
	void setOutput(DS3231_sqw_t mode);
	void enableOutput(bool enabled);
	bool isOutput(void);
	void enable32kHz(bool enabled);
	bool is32kHz(void);

	void forceConversion(void);
	float readTemperature(void);

	void setAlarm1(uint8_t dydw, uint8_t hour, uint8_t minute, uint8_t second, DS3231_alarm1_t mode, bool armed = true);
	RTCAlarmTime getAlarm1(void);
	DS3231_alarm1_t getAlarmType1(void);
	bool isAlarm1(bool clear = true);
	void armAlarm1(bool armed);
	bool isArmed1(void);
	void clearAlarm1(void);

	void setAlarm2(uint8_t dydw, uint8_t hour, uint8_t minute, DS3231_alarm2_t mode, bool armed = true);
	RTCAlarmTime getAlarm2(void);
	DS3231_alarm2_t getAlarmType2(void);
	bool isAlarm2(bool clear = true);
	void armAlarm2(bool armed);
	bool isArmed2(void);
	void clearAlarm2(void);

	void setBattery(bool timeBattery, bool squareBattery);

	char* dateFormat(const char* dateFormat, RTCDateTime dt);
	char* dateFormat(const char* dateFormat, RTCAlarmTime dt);

    private:
	RTCDateTime t;

	char *strDayOfWeek(uint8_t dayOfWeek);
	char *strMonth(uint8_t month);
	char *strAmPm(uint8_t hour, bool uppercase);
	char *strDaySufix(uint8_t day);

	uint8_t hour12(uint8_t hour24);
	uint8_t bcd2dec(uint8_t bcd);
	uint8_t dec2bcd(uint8_t dec);

	long time2long(uint16_t days, uint8_t hours, uint8_t minutes, uint8_t seconds);
	uint16_t date2days(uint16_t year, uint8_t month, uint8_t day);
	uint8_t daysInMonth(uint16_t year, uint8_t month);
	uint16_t dayInYear(uint16_t year, uint8_t month, uint8_t day);
	bool isLeapYear(uint16_t year);
	uint8_t dow(uint16_t y, uint8_t m, uint8_t d);

	uint32_t unixtime(void);
	uint8_t conv2d(const char* p);

	void writeRegister8(uint8_t reg, uint8_t value);
	uint8_t readRegister8(uint8_t reg);
};

#endif

#include <cstdio>
#include <cstdlib>
#include <ctime>
#include <fstream>
#include <iostream>
#include <runner.cuh>
#include <vector>

#define cudaCheck(err) (cudaCheck(err, __FILE__, __LINE__))

const std::string errLogFile = "matrixValidationFailure.txt";

int main(int argc, char **argv) {
  if (argc != 2) {
    std::cerr << "Please select a kernel (range 0 - 12, 0 for NVIDIA cuBLAS)"
              << std::endl;
    exit(EXIT_FAILURE);
  }

  // get kernel number
  int kernel_num = std::stoi(argv[1]);
  if (kernel_num < 0 || kernel_num > 12) {
    std::cerr << "Please enter a valid kernel number (0-12)" << std::endl;
    exit(EXIT_FAILURE);
  }

  // get environment variable for device
  int deviceIdx = 0;
  if (getenv("DEVICE") != NULL) {
    deviceIdx = atoi(getenv("DEVICE"));
  }
  cudaCheck(cudaSetDevice(deviceIdx));

  printf("Running kernel %d on device %d.\n", kernel_num, deviceIdx);

  // print some device info
  // CudaDeviceInfo();

  // Declare the handle, create the handle, cublasCreate will return a value of
  // type cublasStatus_t to determine whether the handle was created
  // successfully (the value is 0)
  cublasHandle_t handle;
  if (cublasCreate(&handle)) {
    std::cerr << "Create cublas handle error." << std::endl;
    exit(EXIT_FAILURE);
  };

  // Using cudaEvent for gpu stream timing, cudaEvent is equivalent to
  // publishing event tasks in the target stream
  float elapsed_time;
  cudaEvent_t beg, end;
  cudaEventCreate(&beg);
  cudaEventCreate(&end);

  // cuBLAS FLOPs ceiling is reached at 8192
  std::vector<int> SIZE = {128, 256, 512, 1024, 2048, 4096};

  long m, n, k, max_size;
  max_size = SIZE[SIZE.size() - 1];
  std::cout << "Max size: " << max_size << std::endl;

  float alpha = 0.5, beta = 3.0; // GEMM input parameters, C=α*AB+β*C

  float *A = nullptr, *B = nullptr, *C = nullptr,
        *C_ref = nullptr; // host matrices
  float *dA = nullptr, *dB = nullptr, *dC = nullptr,
        *dC_ref = nullptr; // device matrices

  A = (float *)malloc(sizeof(float) * max_size * max_size);
  B = (float *)malloc(sizeof(float) * max_size * max_size);
  C = (float *)malloc(sizeof(float) * max_size * max_size);
  C_ref = (float *)malloc(sizeof(float) * max_size * max_size);

  randomize_matrix(A, max_size * max_size);
  randomize_matrix(B, max_size * max_size);
  randomize_matrix(C, max_size * max_size);

  cudaCheck(cudaMalloc((void **)&dA, sizeof(float) * max_size * max_size));
  cudaCheck(cudaMalloc((void **)&dB, sizeof(float) * max_size * max_size));
  cudaCheck(cudaMalloc((void **)&dC, sizeof(float) * max_size * max_size));
  cudaCheck(cudaMalloc((void **)&dC_ref, sizeof(float) * max_size * max_size));

  cudaCheck(cudaMemcpy(dA, A, sizeof(float) * max_size * max_size,
                       cudaMemcpyHostToDevice));
  cudaCheck(cudaMemcpy(dB, B, sizeof(float) * max_size * max_size,
                       cudaMemcpyHostToDevice));
  cudaCheck(cudaMemcpy(dC, C, sizeof(float) * max_size * max_size,
                       cudaMemcpyHostToDevice));
  cudaCheck(cudaMemcpy(dC_ref, C, sizeof(float) * max_size * max_size,
                       cudaMemcpyHostToDevice));

  int repeat_times = 50;
  for (int size : SIZE) {
    m = 32;
    n = k = size;

    std::cout << "dimensions(m,n,k) " << m << "," << n << "," << k << ", alpha: " << alpha
              << ", beta: " << beta << std::endl;
    // Verify the correctness of the calculation, and execute it once before the
    // kernel function timing to avoid cold start errors
    if (kernel_num != 0) {
      run_kernel(0, m, n, k, alpha, dA, dB, beta, dC_ref,
                 handle); // cuBLAS
      run_kernel(kernel_num, m, n, k, alpha, dA, dB, beta, dC,
                 handle); // Executes the kernel, modifies the result matrix
      cudaCheck(cudaDeviceSynchronize());
      cudaCheck(cudaGetLastError()); // Check for async errors during kernel run
      cudaMemcpy(C, dC, sizeof(float) * m * n, cudaMemcpyDeviceToHost);
      cudaMemcpy(C_ref, dC_ref, sizeof(float) * m * n, cudaMemcpyDeviceToHost);

      if (!verify_matrix(C_ref, C, m * n)) {
        std::cout
            << "Failed to pass the correctness verification against NVIDIA "
               "cuBLAS."
            << std::endl;
        if (m <= 128) {
          std::cout << " Logging faulty output into " << errLogFile << "\n";
          std::ofstream fs;
          fs.open(errLogFile);
          fs << "A:\n";
          print_matrix(A, m, n, fs);
          fs << "B:\n";
          print_matrix(B, m, n, fs);
          fs << "C:\n";
          print_matrix(C, m, n, fs);
          fs << "Should:\n";
          print_matrix(C_ref, m, n, fs);
        }
        exit(EXIT_FAILURE);
      }
    }

    cudaEventRecord(beg);
    for (int j = 0; j < repeat_times; j++) {
      // We don't reset dC between runs to save time
      run_kernel(kernel_num, m, n, k, alpha, dA, dB, beta, dC, handle);
    }
    cudaEventRecord(end);
    cudaEventSynchronize(beg);
    cudaEventSynchronize(end);
    cudaEventElapsedTime(&elapsed_time, beg, end);
    elapsed_time /= 1000.; // Convert to seconds

    long flops = 2 * m * n * k;
    printf(
        "Average elapsed time: (%7.6f) s, performance: (%7.1f) GFLOPS. size: "
        "(%ld).\n",
        elapsed_time / repeat_times,
        (repeat_times * flops * 1e-9) / elapsed_time, m);
    fflush(stdout);
    // make dC and dC_ref equal again (we modified dC while calling our kernel
    // for benchmarking)
    cudaCheck(cudaMemcpy(dC, dC_ref, sizeof(float) * m * n,
                         cudaMemcpyDeviceToDevice));
  }

  // Free up CPU and GPU space
  free(A);
  free(B);
  free(C);
  free(C_ref);
  cudaFree(dA);
  cudaFree(dB);
  cudaFree(dC);
  cudaFree(dC_ref);
  cublasDestroy(handle);

  return 0;
};

#include <cuda_runtime.h>
#include <iostream>
#include <vector>

__global__ void kernel(uint *A, uint *B, int row) {
  auto x = threadIdx.x / 4;
  auto y = threadIdx.x % 4;
  A[x * row + y] = x;
  B[x * row + y] = y;
}

int main(int argc, char **argv) {
  uint *Xs, *Ys;
  uint *Xs_d, *Ys_d;

  uint SIZE = 4;

  Xs = (uint *)malloc(SIZE * SIZE * sizeof(uint));
  Ys = (uint *)malloc(SIZE * SIZE * sizeof(uint));

  cudaMalloc((void **)&Xs_d, SIZE * SIZE * sizeof(uint));
  cudaMalloc((void **)&Ys_d, SIZE * SIZE * sizeof(uint));

  dim3 grid_size(1, 1, 1);
  dim3 block_size(4 * 4);

  kernel<<<grid_size, block_size>>>(Xs_d, Ys_d, 4);

  cudaMemcpy(Xs, Xs_d, SIZE * SIZE * sizeof(uint), cudaMemcpyDeviceToHost);
  cudaMemcpy(Ys, Ys_d, SIZE * SIZE * sizeof(uint), cudaMemcpyDeviceToHost);

  cudaDeviceSynchronize();

  for (int row = 0; row < SIZE; ++row) {
    for (int col = 0; col < SIZE; ++col) {
      std::cout << "[" << Xs[row * SIZE + col] << "|" << Ys[row * SIZE + col]
                << "] ";
    }
    std::cout << "\n";
  }

  cudaFree(Xs_d);
  cudaFree(Ys_d);
  free(Xs);
  free(Ys);
}

name: SGEMM_CUDA
channels:
  - conda-forge
dependencies:
  - python
  - black
  - cmake
  - matplotlib
  - seaborn
  - tabulate

.PHONY: all build debug clean profile bench cuobjdump

CMAKE := cmake

BUILD_DIR := build
BENCHMARK_DIR := benchmark_results

all: build

build:
	@mkdir -p $(BUILD_DIR)
	@cd $(BUILD_DIR) && $(CMAKE) -DCMAKE_BUILD_TYPE=Release ..
	@$(MAKE) -C $(BUILD_DIR)

debug:
	@mkdir -p $(BUILD_DIR)
	@cd $(BUILD_DIR) && $(CMAKE) -DCMAKE_BUILD_TYPE=Debug ..
	@$(MAKE) -C $(BUILD_DIR)

clean:
	@rm -rf $(BUILD_DIR)

FUNCTION := $$(cuobjdump -symbols build/sgemm | grep -i Warptiling | awk '{print $$NF}')

cuobjdump: build
	@cuobjdump -arch sm_86 -sass -fun $(FUNCTION) build/sgemm | c++filt > build/cuobjdump.sass
	@cuobjdump -arch sm_86 -ptx -fun $(FUNCTION) build/sgemm | c++filt > build/cuobjdump.ptx

# Usage: make profile KERNEL=<integer> PREFIX=<optional string>
profile: build
	@ncu --set full --export $(BENCHMARK_DIR)/$(PREFIX)kernel_$(KERNEL) --force-overwrite $(BUILD_DIR)/sgemm $(KERNEL)

bench: build
	@bash gen_benchmark_results.sh


[-0.01,  3.00,  3.01, -2.04, -1.02, -3.02,  0.02,  0.02,  4.02,  4.03, -3.01, -3.04,  0.02,  4.01,  4.02, -4.01, -3.03,  1.01, -4.00, -1.04, -0.02,  3.03, -0.03, -1.04,  3.04,  2.03,  3.03, -1.04, -3.01, -3.04,  4.01, -2.01,  3.02,  2.03, -3.03,  1.00,  0.03, -1.01, -2.00,  4.03, -3.04,  2.02,  1.00, -3.04,  2.02,  2.03,  0.00, -4.02, -3.01, -4.02, -2.03,  1.00,  1.03, -3.01, -2.03,  0.00,  3.03, -4.02,  1.02, -1.02,  4.01,  1.02,  3.00,  1.03,  4.01, -0.03,  1.03, -2.02, -1.01,  2.02, -3.01, -0.04,  4.02,  2.04, -3.03, -2.00,  2.01,  4.04,  1.02, -0.01,  3.00, -2.02, -3.04,  1.03, -3.01,  3.03, -2.03, -4.02,  0.03,  0.00, -4.03, -4.00, -1.00,  2.02,  3.02, -4.01, -0.00,  0.01,  1.03,  4.03,  4.00, -0.04,  4.02, -2.04,  0.02, -2.03,  0.04, -2.01,  2.03, -3.02,  4.03,  3.04,  3.00, -1.03, -0.03, -4.03, -4.00,  4.02,  2.02, -4.00, -4.04,  4.04, -0.00,  1.01, -3.03, -0.03,  1.04, -2.01, -0.00,  3.04,  1.03,  2.04,  4.04, -1.02,  2.03,  0.03,  1.01, -4.04,  1.01, -1.02, -2.03,  3.00,  0.04, -0.00, -0.04, -1.04,  0.02,  3.01, -4.03,  0.01,  3.03,  0.01,  1.03,  3.00, -3.00,  1.04,  0.04, -2.04, -1.00,  0.02,  0.01, -2.01, -4.02,  4.02, -2.01,  2.04, -1.01,  2.03,  2.01,  0.02,  1.02,  4.01,  2.00,  1.02,  3.04, -1.01, -2.04,  3.02,  1.01,  0.04,  1.00, -3.04,  3.03, -2.00,  3.00, -3.01,  4.01,  3.01,  1.02,  0.02,  3.00, -1.00,  4.03, -0.02,  1.03,  1.01,  4.02,  4.02,  1.01,  0.01,  4.04,  2.02,  4.01,  4.04, -1.02,  3.03,  4.04,  3.04, -1.00, -1.02, -3.02,  2.01, -0.02, -1.01,  4.02,  0.03,  4.04, -2.03, -2.04, -2.02, -4.00, -3.03, -0.00, -0.03,  4.02,  2.04, -3.00,  2.04, -1.02,  0.01, -0.00,  3.04, -4.01, -2.01,  2.02, -4.01,  3.01, -2.03,  4.04,  3.01, -4.02, -2.00, -4.01,  3.03, -0.01,  2.02, -4.00,  3.03,  1.04,  2.02, -3.03,  2.01,  2.02, -0.02, -4.02, -2.00;
-1.04, -1.04, -2.00, -0.04, -1.00, -0.04,  0.04, -2.04, -2.04, -2.01,  2.00,  1.01, -4.03,  4.00, -0.02, -3.02,  1.04, -4.01, -1.00, -1.04,  4.03, -2.00,  3.04, -2.00,  1.00,  1.01,  1.02, -2.04,  1.00,  2.03,  0.03,  1.02,  3.00, -3.03,  0.01, -0.04, -3.03, -1.00,  3.03, -2.03,  0.03,  2.02, -4.00, -0.00,  3.00,  2.02, -3.03, -2.04, -2.04, -3.01,  1.04, -2.01, -2.01,  0.03,  1.04,  4.03,  2.01,  0.01,  4.03,  0.03, -4.04, -4.03, -4.00, -0.03, -0.01,  4.00, -2.00,  2.00,  3.01,  0.01, -4.00, -2.02,  2.01,  1.03, -4.03, -1.04, -3.02,  2.01,  3.01,  0.01, -1.01, -1.03, -3.02, -4.04,  2.03,  1.02,  1.02, -0.01, -2.02,  4.02, -4.03, -1.01,  1.00, -3.01, -4.01,  4.02,  2.03,  2.01, -0.03, -4.04,  1.00, -3.02, -2.03,  0.03, -3.03, -1.04, -3.01, -2.00, -3.02, -1.01, -2.02,  0.02, -2.01,  4.02, -2.00, -3.01,  4.04,  3.01,  1.02, -2.03, -2.02, -2.02,  2.02,  4.01,  2.00, -3.03, -1.04, -4.03,  0.01, -0.03, -3.00,  4.00, -4.02, -0.01,  4.01, -2.04,  1.03, -3.02, -4.00, -2.00,  4.04, -0.03,  3.02, -1.02, -1.03, -2.01, -0.04, -0.02,  3.02,  0.04, -0.03,  2.00,  0.04,  4.02, -4.03, -0.02, -1.00, -4.00, -2.01, -3.01,  2.01, -4.03,  4.00,  0.02, -1.01,  1.04,  3.00, -0.01, -1.04,  4.04, -3.01,  1.03,  2.02,  1.04, -2.01, -1.04, -2.02, -4.04, -0.00, -0.01, -0.00, -1.01,  0.03,  2.02,  1.04, -4.00, -0.04,  2.03, -2.03, -1.00, -1.02, -3.03, -0.01, -1.03, -2.02,  3.03, -1.03, -3.00, -2.02,  1.00,  0.02,  0.00,  2.01,  0.02, -1.01, -4.02, -1.03,  0.01, -3.00, -3.02,  3.04, -3.04,  0.04, -4.04, -4.02,  1.01, -1.00, -2.02, -3.02, -3.00, -4.01,  2.04,  3.01,  4.01,  2.00,  2.02,  1.00, -3.04, -0.04, -0.03,  2.02,  0.02, -3.00,  4.01, -2.04,  4.03,  2.04, -0.04, -3.00,  0.00, -3.00, -2.00, -3.03,  0.00,  3.04, -3.03, -1.02, -1.02, -0.00,  1.03,  0.01,  2.01, -4.01, -0.01,  4.01,  4.04;
 2.02, -3.03,  2.01, -3.02, -2.03, -0.02, -1.01,  3.03,  1.04, -4.00,  2.04, -0.03,  1.00,  4.01,  2.03, -4.00, -2.04,  0.04,  2.00, -4.01,  0.04, -4.01, -1.04,  1.00,  0.02,  0.04, -3.03, -3.03, -1.02, -3.04,  0.04,  4.03,  3.01,  3.04, -0.02, -1.02,  1.02, -3.01,  3.04,  3.02,  2.04,  0.04, -2.02, -3.02, -1.03,  4.01, -1.02,  1.02, -2.00,  0.00, -2.03, -0.03, -2.00,  1.00, -1.04, -0.03,  1.02, -4.03, -1.02, -1.04, -1.03, -0.04,  3.01,  2.04,  2.02, -2.02, -3.03,  1.00,  1.01, -3.04, -3.02,  3.02,  4.02, -1.00,  0.02,  0.00, -1.00, -2.02, -3.00,  3.01,  3.04,  2.02, -4.02, -3.00,  4.02,  2.02,  2.00, -3.04,  4.04, -1.03, -0.01,  1.03,  1.00,  4.00,  2.04,  3.03, -2.01,  0.01, -2.01,  2.04,  0.01, -0.02, -4.01,  0.00,  0.03, -2.03,  1.00,  4.03, -2.02,  0.01,  0.04, -1.04,  1.00,  2.02, -4.02, -3.04, -0.00,  0.01, -3.04,  1.04, -0.01,  0.03, -0.01,  1.00, -0.00,  1.00, -3.02, -0.04, -4.02,  0.03, -3.00, -0.01,  3.01, -3.00, -3.00,  2.03, -0.01,  0.03,  0.02, -0.03, -2.03, -3.04,  4.04, -3.03,  2.03, -4.01,  2.01,  3.02, -2.04,  1.01, -4.03,  4.01, -0.02,  2.02, -4.02,  3.03, -3.00,  0.00,  0.00, -4.01,  4.02,  4.01,  3.01,  1.01,  0.00,  2.00,  0.04, -1.01,  0.01, -4.02, -0.03, -0.02,  3.00, -2.04,  2.04, -1.01,  3.01, -0.02,  2.04, -4.04, -1.04, -1.01, -0.01,  0.04,  0.03,  0.00,  0.00,  2.00, -4.04, -0.00, -0.02,  3.00,  1.00,  2.02,  1.02, -2.02, -3.00, -3.02,  2.03,  0.00,  0.00, -4.01, -4.04,  3.03, -4.00,  1.02, -4.00, -3.03, -0.00, -1.03,  4.03, -0.02, -3.03,  1.02,  1.03, -4.04,  1.04,  3.04, -4.04, -3.02,  0.04, -3.01,  0.04,  0.02, -1.04,  1.03,  1.04,  1.04,  3.03,  1.03, -4.04,  0.04,  3.04,  2.02, -2.03,  4.04, -0.00, -4.03, -3.02, -4.00, -4.02,  1.02, -1.03,  0.00, -3.02, -1.04,  3.04,  2.00,  4.04, -0.00, -4.04,  4.02,  3.03,  3.00,  1.00,  1.01;
 3.02, -1.04,  2.00,  4.00, -1.01,  1.00, -0.00,  2.01, -4.04, -2.02, -0.01, -3.04,  4.03, -4.02, -1.00, -2.01, -0.01,  3.00,  3.01, -4.04, -3.01, -4.01, -1.01, -1.00,  1.02, -3.01,  2.04, -1.04,  4.03, -2.04,  3.02,  4.01, -2.02, -0.00, -0.04,  3.02, -2.01, -4.02, -2.03,  2.03, -2.03, -4.03, -4.02,  3.03,  4.00, -2.03,  4.00,  2.02, -4.04,  0.03,  1.03, -2.02,  0.03, -1.01, -1.02,  0.04, -3.00,  2.02,  0.02,  4.04, -3.02, -1.04,  0.02, -0.00,  0.01, -3.03,  4.01,  2.00,  3.00, -3.02,  0.00, -4.03, -1.04, -2.01, -0.02, -1.02,  4.01,  2.00,  4.03,  1.03,  1.04,  4.03,  4.01,  2.03,  1.04, -2.02, -2.01,  4.00, -0.02,  0.03,  2.04, -1.01,  4.00, -1.04,  2.02, -1.00, -3.02,  2.01,  2.03, -1.00, -0.04, -3.01, -3.00,  2.04, -2.03,  1.04, -0.04,  2.02, -4.04,  4.00, -1.02, -0.02,  3.00, -0.03,  2.00,  3.04, -4.04,  2.01, -0.02,  2.01, -2.03,  1.00,  3.01,  0.01, -2.00,  3.00,  1.04,  4.01, -0.02,  4.01,  3.03,  4.01,  2.04, -3.00, -2.04, -2.04, -2.01,  4.02,  3.02,  0.04, -4.04,  2.00,  1.02, -3.03,  2.00,  3.04,  4.04,  0.00,  3.03,  1.01,  4.00, -0.04,  2.03,  3.01, -2.03,  4.01,  3.01,  3.01,  0.01,  4.03,  4.04, -3.04,  4.01, -0.02, -4.04, -3.00,  1.04, -3.02, -2.00,  1.04,  2.00, -2.03, -2.03, -2.00, -2.02, -4.00,  0.01,  2.01,  0.00, -4.01, -4.03,  3.03,  3.02, -2.00,  3.04,  3.03, -1.00, -2.02,  3.02, -1.02, -3.03,  1.03, -4.03, -2.01,  0.04, -1.00, -1.01,  0.00, -4.01, -4.02,  2.03,  3.04, -0.00, -4.02, -3.04, -3.00,  1.04, -4.04, -0.03,  1.04,  0.01,  3.02,  0.04,  3.00,  2.03, -3.03, -3.04, -2.00, -3.03,  0.04, -3.01,  3.04,  0.02,  3.00, -4.03, -0.03, -4.01,  1.00, -3.03, -2.00,  3.04, -0.04,  1.01,  3.04, -0.04, -2.01, -2.00, -1.02,  3.02, -2.04, -3.04, -3.02, -1.03, -2.04,  0.04,  3.02,  1.01,  3.00,  3.03, -2.02, -3.00,  0.00,  3.02,  2.01,  1.03,  4.03;

cmake_minimum_required(VERSION 3.19)
project(NVIDIA_SGEMM_PRACTICE LANGUAGES CXX CUDA)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

find_package(CUDA REQUIRED)

# ensure cuda is available
include(CheckLanguage)
check_language(CUDA)

set(CMAKE_CXX_STANDARD 20)
set(CUDA_COMPUTE_CAPABILITY 86)

# in debug mode, add debug symbols to device code
# this disables most optimizations and kills performance
add_compile_options("$<$<AND:$<CONFIG:Debug>,$<COMPILE_LANGUAGE:CUDA>>:-G;-src-in-ptx>")
# add_compile_options("--ptxas-options=-v")

# Configure header file search paths
include_directories(${CUDA_INCLUDE_DIRS})
include_directories(${PROJECT_SOURCE_DIR}/src)
# Configure the source file path to be compiled
aux_source_directory(${PROJECT_SOURCE_DIR}/src SRC)

# generate executable
add_executable(sgemm sgemm.cu ${SRC})
set_target_properties(sgemm PROPERTIES CUDA_ARCHITECTURES ${CUDA_COMPUTE_CAPABILITY})
target_link_libraries(sgemm ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES})

add_executable(cuBLAS_sgemm cuBLAS_sgemm.cu )
set_target_properties(sgemm PROPERTIES CUDA_ARCHITECTURES ${CUDA_COMPUTE_CAPABILITY})
target_link_libraries(cuBLAS_sgemm ${CUDA_LIBRARIES} ${CUDA_CUBLAS_LIBRARIES})

add_executable(simplest_kernel simplest_kernel.cu)
set_target_properties(sgemm PROPERTIES CUDA_ARCHITECTURES ${CUDA_COMPUTE_CAPABILITY})
target_link_libraries(simplest_kernel ${CUDA_LIBRARIES})

import gzip
import random
import tqdm
import numpy as np

import torch
from torch.optim import Adam
from torch.nn import functional as F
from torch.utils.data import DataLoader, Dataset

from simple_hierarchical_transformer import HierarchicalTransformer

# constants

NUM_BATCHES = int(1e5)
BATCH_SIZE = 2
GRADIENT_ACCUMULATE_EVERY = 8
LEARNING_RATE = 1e-4
VALIDATE_EVERY = 100
PRIME_LENGTH = 128
GENERATE_EVERY = 500
SEQ_LEN = 8192
GENERATE_LENGTH = 1024

# helpers

def cycle(loader):
    while True:
        for data in loader:
            yield data

def decode_token(token):
    return str(chr(max(32, token)))

def decode_tokens(tokens):
    return "".join(list(map(decode_token, tokens)))

# instantiate transformer

model = HierarchicalTransformer(
    num_tokens = 256,
    dim = 1024,
    depth = 8,
    seq_len = SEQ_LEN,
    use_flash_attn = True,
    compress_factor = 32
).cuda()

# prepare enwik8 data

with gzip.open("./data/enwik8.gz") as file:
    data = np.frombuffer(file.read(int(95e6)), dtype=np.uint8).copy()
    np_train, np_valid = np.split(data, [int(90e6)])
    data_train, data_val = torch.from_numpy(np_train), torch.from_numpy(np_valid)

class TextSamplerDataset(Dataset):
    def __init__(self, data, seq_len):
        super().__init__()
        self.data = data
        self.seq_len = seq_len

    def __getitem__(self, index):
        rand_start = torch.randint(0, self.data.size(0) - self.seq_len, (1,))
        full_seq = self.data[rand_start : rand_start + self.seq_len + 1].long()
        return full_seq.cuda()

    def __len__(self):
        return self.data.size(0) // self.seq_len

train_dataset = TextSamplerDataset(data_train, SEQ_LEN)
val_dataset = TextSamplerDataset(data_val, SEQ_LEN)
train_loader = cycle(DataLoader(train_dataset, batch_size=BATCH_SIZE))
val_loader = cycle(DataLoader(val_dataset, batch_size=BATCH_SIZE))

# optimizer

optim = Adam(model.parameters(), lr = LEARNING_RATE)

# training

for i in tqdm.tqdm(range(NUM_BATCHES), mininterval = 10.0, desc = "training"):
    model.train()

    for _ in range(GRADIENT_ACCUMULATE_EVERY):
        loss, (ce_loss, recon_loss) = model(next(train_loader), return_loss = True)
        loss.backward(loss / GRADIENT_ACCUMULATE_EVERY)

    print(f"training loss: {ce_loss.item()}")
    torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)

    optim.step()
    optim.zero_grad()

    if i % VALIDATE_EVERY == 0:
        model.eval()
        with torch.no_grad():
            loss, (ce_loss, recon_loss) = model(next(val_loader), return_loss = True)
            print(f"validation loss: {ce_loss.item()}")

    if i % GENERATE_EVERY == 0:
        model.eval()
        inp = random.choice(val_dataset)[:PRIME_LENGTH]
        prime = decode_tokens(inp)
        print(f"%s \n\n %s", (prime, "*" * 100))

        sample = model.generate(inp[None, ...], GENERATE_LENGTH)
        output_str = decode_tokens(sample[0])
        print(output_str, "\n")


{
  "version": "1.0",
  "truncation": null,
  "padding": null,
  "added_tokens": [
    {
      "id": 0,
      "content": "<unk>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": true,
      "special": true
    },
    {
      "id": 1,
      "content": "<s>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": true,
      "special": true
    },
    {
      "id": 2,
      "content": "</s>",
      "single_word": false,
      "lstrip": false,
      "rstrip": false,
      "normalized": true,
      "special": true
    }
  ],
  "normalizer": {
    "type": "Sequence",
    "normalizers": [
      {
        "type": "Prepend",
        "prepend": "▁"
      },
      {
        "type": "Replace",
        "pattern": {
          "String": " "
        },
        "content": "▁"
      }
    ]
  },
  "pre_tokenizer": null,
  "post_processor": {
    "type": "TemplateProcessing",
    "single": [
      {
        "SpecialToken": {
          "id": "<s>",
          "type_id": 0
        }
      },
      {
        "Sequence": {
          "id": "A",
          "type_id": 0
        }
      }
    ],
    "pair": [
      {
        "SpecialToken": {
          "id": "<s>",
          "type_id": 0
        }
      },
      {
        "Sequence": {
          "id": "A",
          "type_id": 0
        }
      },
      {
        "SpecialToken": {
          "id": "<s>",
          "type_id": 1
        }
      },
      {
        "Sequence": {
          "id": "B",
          "type_id": 1
        }
      }
    ],
    "special_tokens": {
      "<s>": {
        "id": "<s>",
        "ids": [
          1
        ],
        "tokens": [
          "<s>"
        ]
      }
    }
  },
  "decoder": {
    "type": "Sequence",
    "decoders": [
      {
        "type": "Replace",
        "pattern": {
          "String": "▁"
        },
        "content": " "
      },
      {
        "type": "ByteFallback"
      },
      {
        "type": "Fuse"
      },
      {
        "type": "Strip",
        "content": " ",
        "start": 1,
        "stop": 0
      }
    ]
  }
}

#include "q_gemm.cuh"
#include "util.cuh"
#include "matrix_view.cuh"
#include "../config.h"

// #include <cuda/pipeline>
// #include <mma.h>

#define BLOCK_KN_SIZE 512
#define MAX_GROUPS_IN_BLOCK (BLOCK_KN_SIZE / 32)
#define MAX_COUNT_M 2
#define CLEAR_N_SIZE 256
//#define DEBUG

#include "q_gemm_dq.cuh"

typedef void (*fp_gemm_half_q_half_kernel)
(
    const half*,
    const uint32_t*,
    const uint32_t*,
    const half*,
    const uint16_t*,
    half*,
    const int,
    const int,
    const int,
    const int,
    const int,
    const uint16_t*,
    const int,
    const int,
    const int,
    const int,
    const int,
    const int
);

template <bool first_block, int m_count>
__global__ void gemm_half_q_half_kernel
(
    const half*      a,
    const uint32_t*  b_q_weight,
    const uint32_t*  b_q_scale,
    const half*      b_q_scale_max,
    const uint16_t*  b_q_groups,
    half*            c,
    const int size_m,
    const int size_n,
    const int size_k,
    const int groups,
    const int groupsize,
    const uint16_t* __restrict__ b_q_perm,
    const int rows_8,
    const int rows_6,
    const int rows_5,
    const int rows_4,
    const int rows_3,
    const int rows_2
)
{
    MatrixView_half a_(a, size_m, size_k);
    MatrixView_half_rw c_(c, size_m, size_n);
    MatrixView_q4_row b_q_scale_(b_q_scale, groups, size_n);

    int t = threadIdx.x;

    // Block

    int offset_n = blockIdx.x * BLOCK_KN_SIZE;
    int offset_m = blockIdx.y * m_count;
    int offset_k = blockIdx.z * BLOCK_KN_SIZE;

    int end_n = min(offset_n + BLOCK_KN_SIZE, size_n);
    int end_m = min(offset_m + m_count, size_m);
    int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
    int n = offset_n + t;

    // Preload block_a

    __shared__ half block_a[m_count][BLOCK_KN_SIZE];

    if (offset_k + t < end_k)
    {
        for (int m = 0; m < m_count; ++m)
        {
            const half* a_ptr = a_.item_ptr(offset_m + m, 0);
            half* block_a_ptr = block_a[m];
            half a0 = a_ptr[b_q_perm[offset_k + t]];
            block_a_ptr[t] = a0;
        }
    }

    __syncthreads();

    if (n >= size_n) return;

//     for (int xs_n = 0; xs_n < size_n - BLOCK_KN_SIZE;

    // Advance to subblock

//     int sub_offset_k = SUBBLOCK_K_SIZE * threadIdx.y;
//     offset_k += sub_offset_k;
//     end_k = min(offset_k + SUBBLOCK_K_SIZE, size_k);

//     if (threadIdx.y > 0) return;
//     int sub_offset_k = 0;

    // Find initial group

    int group = offset_k / groupsize;

    // Preload scales

    half scales[MAX_GROUPS_IN_BLOCK];

    int groups_in_block = DIVIDE((end_k - offset_k), groupsize);
    for (int g = 0; g < groups_in_block; g++)
    {
        half s = dq_scale(b_q_scale_.item(group + g, n), b_q_scale_max[group + g]);
        scales[g] = s;
    }

    // Find initial q row

    int pre_rows_8 = min(rows_8, offset_k);
    int pre_rows_6 = offset_k > rows_8 ? min(rows_6, offset_k) - rows_8 : 0;
    int pre_rows_5 = offset_k > rows_6 ? min(rows_5, offset_k) - rows_6 : 0;
    int pre_rows_4 = offset_k > rows_5 ? min(rows_4, offset_k) - rows_5 : 0;
    int pre_rows_3 = offset_k > rows_4 ? min(rows_3, offset_k) - rows_4 : 0;
    int pre_rows_2 = offset_k > rows_3 ? min(rows_2, offset_k) - rows_3 : 0;
    int qk = 0;
    qk += pre_rows_8 / 32 * 8;
    qk += pre_rows_6 / 32 * 6;
    qk += pre_rows_5 / 32 * 5;
    qk += pre_rows_4 / 32 * 4;
    qk += pre_rows_3 / 32 * 3;
    qk += pre_rows_2 / 32 * 2;

    const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
    const half* a_ptr = &block_a[0][0];
    int a_stride = BLOCK_KN_SIZE;
//     const half* a_ptr = a_.item_ptr(offset_m, 0);
//     int a_stride = size_k;

    half qs_h = scales[0];
    int scales_idx = 0;
    int nextgroup = offset_k + groupsize;

    // Column result

    half2 block_c[m_count] = {};

    // Dot product over groups

    int k = offset_k;

    while (k < rows_8 && k < end_k)
    {
        int end_k_sg = min(min(k + 128, rows_6), end_k);
        uint32_t q_0[8], q_1[8];
        load_8(b_ptr, size_n, q_0);
        qdot_8bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0, q_1);
        qdot_8bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1, q_0);
        qdot_8bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0, q_1);
        qdot_8bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1, q_0);
    }

    while (k < rows_6 && k < end_k)
    {
        int end_k_sg = min(min(k + 128, rows_5), end_k);
        uint32_t q_0[6], q_1[6];
        load_6(b_ptr, size_n, q_0);
        qdot_6bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0, q_1);
        qdot_6bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1, q_0);
        qdot_6bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0, q_1);
        qdot_6bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1, q_0);
    }

    while (k < rows_5 && k < end_k)
    {
        int end_k_sg = min(min(k + 128, rows_4), end_k);
        uint32_t q_0[5], q_1[5];
        load_5(b_ptr, size_n, q_0);
        qdot_5bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0, q_1);
        qdot_5bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1, q_0);
        qdot_5bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0, q_1);
        qdot_5bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1, q_0);
    }

//     while (k + 128 < rows_4 && k + 128 < end_k)
//     {
//         uint32_t q_0[8], q_1[8];
//         load_8(b_ptr, size_n, q_0);
//         load_8(b_ptr, size_n, q_1);
//         qdot_4bit_64<m_count>(k, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0);
//         qdot_4bit_64<m_count>(k, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1);
//     }

    while (k < rows_4 && k < end_k)
    {
        int end_k_sg = min(min(k + 128, rows_3), end_k);
        uint32_t q_0[4], q_1[4];
        load_4(b_ptr, size_n, q_0);
        qdot_4bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0, q_1);
        qdot_4bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1, q_0);
        qdot_4bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0, q_1);
        qdot_4bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1, q_0);
    }

//     while (k + 128 < rows_3 && k + 128 < end_k)
//     {
//         uint32_t q_0[6], q_1[6];
//         load_6(b_ptr, size_n, q_0);
//         load_6(b_ptr, size_n, q_1);
//         qdot_3bit_64<m_count>(k, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0);
//         qdot_3bit_64<m_count>(k, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1);
//     }

    while (k < rows_3 && k < end_k)
    {
        int end_k_sg = min(min(k + 128, rows_2), end_k);
        uint32_t q_0[3], q_1[3];
        load_3(b_ptr, size_n, q_0);
        qdot_3bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0, q_1);
        qdot_3bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1, q_0);
        qdot_3bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0, q_1);
        qdot_3bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1, q_0);
    }

//     while (k + 128 < rows_2 && k + 128 < end_k)
//     {
//         uint32_t q_0[8];
//         load_8(b_ptr, size_n, q_0);
//         qdot_2bit_128<m_count>(k, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0);
//     }

    while (k < rows_2 && k < end_k)
    {
        int end_k_sg = min(k + 128, end_k);
        uint32_t q_0[2], q_1[2];
        load_2(b_ptr, size_n, q_0);
        qdot_2bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0, q_1);
        qdot_2bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1, q_0);
        qdot_2bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_0, q_1);
        qdot_2bit_32<m_count>(k, end_k_sg, group, nextgroup, groupsize, n, scales, scales_idx, qs_h, block_c, a_ptr, a_stride, b_ptr, size_n, q_1, q_0);
    }

    // Accumulate column sums in c

    for (int m = 0; m < m_count; m++) atomicAdd(c_.item_ptr(offset_m + m, n), __hadd(block_c[m].x, block_c[m].y));
    //for (int m = 0; m < m_count; m++) c_.set(offset_m + m, n, block_c[m]);
}

fp_gemm_half_q_half_kernel pick_gemm_half_q_half_kernel(bool first_block, const int m_count)
{
    if (m_count == 1) return gemm_half_q_half_kernel<true, 1>;
    if (m_count == 2) return gemm_half_q_half_kernel<true, 2>;
//     if (m_count == 3) return gemm_half_q_half_kernel<true, 3>;
//     if (m_count == 4) return gemm_half_q_half_kernel<true, 4>;
//     if (m_count == 5) return gemm_half_q_half_kernel<true, 5>;
//     if (m_count == 6) return gemm_half_q_half_kernel<true, 6>;
//     if (m_count == 7) return gemm_half_q_half_kernel<true, 7>;
//     if (m_count == 8) return gemm_half_q_half_kernel<true, 8>;
    return NULL;
}

void gemm_half_q_half_cuda_part
(
    const half* a,
    QMatrix* b,
    half* c,
    int size_m,
    int size_n,
    int size_k,
    int count_m
)
{
    dim3 blockDim, gridDim;
    blockDim.x = BLOCK_KN_SIZE;
    blockDim.y = 1;
    blockDim.z = 1;
    gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE);
    gridDim.y = DIVIDE(size_m, count_m);
    gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);

    fp_gemm_half_q_half_kernel kernel = pick_gemm_half_q_half_kernel(true, count_m);

    kernel<<<gridDim, blockDim>>>
    (
        a,
        b->cuda_q_weight,
        b->cuda_q_scale,
        b->cuda_q_scale_max,
        b->cuda_q_groups,
        c,
        size_m,
        size_n,
        size_k,
        b->groups,
        b->groupsize,
        b->cuda_q_perm,
        b->rows_8,
        b->rows_6,
        b->rows_5,
        b->rows_4,
        b->rows_3,
        b->rows_2
    );
}

void gemm_half_q_half_cuda
(
    cublasHandle_t cublas_handle,
    const half* a,
    QMatrix* b,
    half* c,
    int size_m,
    int size_n,
    int size_k,
    bool clear,
    half* temp_dq
)
{
    if (size_m >= MAX_Q_GEMM_ROWS)
    {
        // Reconstruct FP16 matrix, then cuBLAS

        //DBGI3(size_m, size_n, size_k);

        if (!temp_dq) temp_dq = b->temp_dq;
        b->reconstruct(temp_dq);

        cublasSetMathMode(cublas_handle, CUBLAS_TENSOR_OP_MATH);

        //DBGI3(size_m, size_n, size_k);

        const half alpha = __float2half(1.0f);
        const half beta = clear ? __float2half(0.0f) : __float2half(1.0f);
        cublasHgemm(cublas_handle,
                    CUBLAS_OP_N,
                    CUBLAS_OP_N,
                    size_n,
                    size_m,
                    size_k,
                    &alpha,
                    temp_dq,
                    size_n,
                    a,
                    size_k,
                    &beta,
                    c,
                    size_n);

//         const float alpha = 1.0f;
//         const float beta = clear ? 0.0f : 1.0f;
//         cublasSgemmEx(cublas_handle,
//                       CUBLAS_OP_N,
//                       CUBLAS_OP_N,
//                       size_n,
//                       size_m,
//                       size_k,
//                       &alpha,
//                       temp_dq,
//                       CUDA_R_16F,
//                       size_n,
//                       a,
//                       CUDA_R_16F,
//                       size_k,
//                       &beta,
//                       c,
//                       CUDA_R_16F,
//                       size_n);
    }
    else
    {
        // Quantized matmul

        if (clear) clear_tensor_cuda(c, size_m, size_n);

        int max_chunks = size_m / MAX_COUNT_M;
        int last_chunk = max_chunks * MAX_COUNT_M;
        int last_chunk_size = size_m - last_chunk;

    //     DBGI3(size_m, size_n, size_k);
    //     DBGI3(max_chunks, last_chunk, last_chunk_size);

        if (max_chunks)
            gemm_half_q_half_cuda_part(a, b, c, last_chunk, size_n, size_k, MAX_COUNT_M);

        if (last_chunk_size)
            gemm_half_q_half_cuda_part(a + last_chunk * size_k, b, c + last_chunk * size_n, last_chunk_size, size_n, size_k, last_chunk_size);
    }
}

__global__ void clear_kernel
(
    half* __restrict__ c,
    const int size_m,
    const int size_n
)
{
    int m = blockIdx.y;
    int n = (blockIdx.x * CLEAR_N_SIZE + threadIdx.x) * 8;
    if (n >= size_n) return;
    int4* c_ptr = (int4*)(c + m * size_n + n);
    *c_ptr = {};
}

void clear_tensor_cuda
(
    half* c,
    int size_m,
    int size_n
)
{
    dim3 blockDim, gridDim;
    blockDim.x = CLEAR_N_SIZE;
    blockDim.y = 1;
    gridDim.x = DIVIDE(size_n / 8, CLEAR_N_SIZE);
    gridDim.y = size_m;
    clear_kernel<<<gridDim, blockDim>>>(c, size_m, size_n);
}

#!/usr/bin/env python3
import argparse

import torch
import torch.nn as nn
from datasets import load_dataset
from gptq_triton import load_quant
from tqdm import tqdm
from transformers import AutoTokenizer, LlamaForCausalLM


parser = argparse.ArgumentParser()

parser.add_argument('--model', type=str, help='Path to model, either a HuggingFace model or a quantized model')
parser.add_argument('--quant', action='store_true', help='Whether the model is quantized')
parser.add_argument('--stride', type=int, default=512, help='Stride for calculating perplexity')
parser.add_argument('--context-length', type=int, default=2048, help='Length of context to use')


def main():
	args = parser.parse_args()

	if not args.quant:
		model = get_llama(args.model)
		model.eval()
		model.to('cuda')
	else:
		model = load_quant(args.model)
		model.eval()
		model.to('cuda')

	# NOTE: Setting use_fast=False for now, as the alternative was an order of magnitude slower on a recent `transformers` commit
	tokenizer = AutoTokenizer.from_pretrained(args.model, use_fast=False)
	context_length = model.seqlen if args.context_length is None else args.context_length

	for dataset in ['wikitext-2', 'ptb', 'c4']:
		ppl = calculate_perplexity(model, tokenizer, dataset, max_length=context_length, stride=args.stride)
		print(f"{dataset} perplexity: {ppl}")


def get_llama(model: str):
	"""
	Load a pretrained Llama model
	"""
	def skip(*args, **kwargs):
		pass
	# NOTE: This is a nasty hack, but it speeds up model building by a huge amount
	old_inits = (torch.nn.init.kaiming_uniform_, torch.nn.init.uniform_, torch.nn.init.normal_)
	torch.nn.init.kaiming_uniform_ = skip
	torch.nn.init.uniform_ = skip
	torch.nn.init.normal_ = skip

	model = LlamaForCausalLM.from_pretrained(model, torch_dtype='auto')
	model.seqlen = 2048

	# Restore the old initializers
	torch.nn.init.kaiming_uniform_, torch.nn.init.uniform_, torch.nn.init.normal_ = old_inits

	return model


def get_dataset(dataset_name: str, tokenizer) -> torch.Tensor:
	if dataset_name == "wikitext-2":
		test = load_dataset("wikitext", "wikitext-2-raw-v1", split="test")
		encodings = tokenizer("\n\n".join(test["text"]), return_tensors="pt").input_ids
	elif dataset_name == 'ptb':
		test = load_dataset("ptb_text_only", 'penn_treebank', split="validation")
		encodings = tokenizer("\n\n".join(test["sentence"]), return_tensors="pt").input_ids
	elif dataset_name == 'c4':
		# WARNING: Many of the files in the allenai/c4 repo are marked as "Unsafe" by HuggingFace, possibly containing a virus.  This particular file is not, and I doubt it's an issue, but worth noting.
		test = load_dataset('allenai/c4', 'allenai--c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation')
		encodings = [tokenizer(x, return_tensors="pt").input_ids for x in test['text'][:1000]]
		encodings = torch.cat(encodings, dim=1)
	else:
		raise ValueError(f"Unknown dataset {dataset_name}")

	return encodings


def calculate_perplexity(model, tokenizer, dataset: str, max_length: int, stride: int = 512) -> float:
	print("Loading dataset...")
	encodings = get_dataset(dataset, tokenizer)
	seq_len = encodings.size(1)

	print("Calculating perplexity...")
	print(f"Sequence length: {seq_len}")
	print(f"Max length: {max_length}")
	print(f"Stride: {stride}")

	nlls = []
	prev_end_loc = 0

	for begin_loc in (pbar := tqdm(range(0, seq_len - 1, stride))):
		end_loc = min(seq_len - 1, begin_loc + max_length)
		trg_len = end_loc - prev_end_loc  # How many tokens we want to predict
		input_ids = encodings[:, begin_loc:end_loc+1].to('cuda')  # +1 for the labels

		with torch.no_grad():
			# Ask the model for logits
			# NOTE: Instead of calling HF's model wrapper, we call the model directly to hopefully cut down on some memory overhead
			outputs = model.model(input_ids[:, :-1], use_cache=False)
			logits = model.lm_head(outputs[0][..., -trg_len:, :])

			# The last trg_len tokens are the labels
			labels = input_ids[:, -trg_len:].contiguous()

			# Compute the NLL for this batch using flattened logits and labels
			loss_fct = nn.CrossEntropyLoss()
			loss = loss_fct(logits.view(-1, logits.size(-1)), labels.view(-1))

		nlls.append(loss.to('cpu').to(torch.float32))
		ppl = torch.exp(torch.stack(nlls).mean())
		pbar.set_description(f"Perplexity: {ppl:.2f}")

		prev_end_loc = end_loc
		if end_loc == (seq_len - 1):
			break

	ppl = torch.exp(torch.stack(nlls).mean())

	return ppl


if __name__ == '__main__':
	main()


[metadata]
name = gptq_triton
version = 0.0.3
author = fpgaminer
author_email = fpgaminer@bitcoin-mining.com
description = Fast GPTQ kernels written in Triton
long_description = file: README.md
long_description_content_type = text/markdown; charset=UTF-8
url = https://github.com/fpgaminer/GPTQ-triton
keywords = gptq, triton, torch, cuda, gpu, quantization, quantize, quantized, inference, deep learning, machine learning
license = Apache License 2.0
license_file = LICENSE
classifiers =
	Development Status :: 3 - Alpha
	License :: OSI Approved :: Apache Software License
	Intended Audience :: Developers
	Programming Language :: Python :: 3
	Programming Language :: Python :: 3.6
	Programming Language :: Python :: 3.7
	Programming Language :: Python :: 3.8
	Programming Language :: Python :: 3.9
	Programming Language :: Python :: 3.10
	Topic :: Scientific/Engineering :: Artificial Intelligence
	Topic :: Software Development :: Libraries :: Python Modules

[options]
zip_safe = False
include_package_data = False
package_dir =
	= src
packages = find:
python_requires = >=3.6
install_requires =
	triton >= 2.0.0
	torch >= 2.0.0
	transformers

[options.packages.find]
where = src

import random

from datasets import load_dataset


def get_dataset(dataset_name: str, tokenizer, nsamples: int, seed: int, seqlen: int):
	if dataset_name == "wikitext-2":
		return get_wikitext2(nsamples, seed, seqlen, tokenizer)
	elif dataset_name == 'ptb':
		return get_ptb(nsamples, seed, seqlen, tokenizer, jointext='\n\n')
	elif dataset_name == 'ptb-new':
		return get_ptb(nsamples, seed, seqlen, tokenizer, jointext=' ')
	elif dataset_name == 'c4':
		return get_c4(nsamples, seed, seqlen, tokenizer)
	else:
		raise ValueError(f"Unknown dataset {dataset_name}")


def get_wikitext2(nsamples: int, seed: int, seqlen: int, tokenizer, jointext: str = '\n\n'):
	traindata = load_dataset('wikitext', 'wikitext-2-raw-v1', split='train')

	trainenc = tokenizer(jointext.join(traindata['text']), return_tensors='pt')

	rng = random.Random(seed)
	trainloader = (rng.randint(0, trainenc.input_ids.shape[1] - seqlen - 1) for _ in range(nsamples))
	trainloader = [trainenc.input_ids[:, i:i+seqlen] for i in trainloader]

	return trainloader


def get_ptb(nsamples: int, seed: int, seqlen: int, tokenizer, jointext: str):
	traindata = load_dataset('ptb_text_only', 'penn_treebank', split='train')

	trainenc = tokenizer(jointext.join(traindata['sentence']), return_tensors='pt')

	rng = random.Random(seed)
	trainloader = (rng.randint(0, trainenc.input_ids.shape[1] - seqlen - 1) for _ in range(nsamples))
	trainloader = [trainenc.input_ids[:, i:i+seqlen] for i in trainloader]

	return trainloader


def get_c4(nsamples: int, seed: int, seqlen: int, tokenizer):
	# WARNING: Many of the files in the allenai/c4 repo are marked as "Unsafe" by HuggingFace, possibly containing a virus.  This particular file is not, and I doubt it's an issue, but worth noting.
	traindata = load_dataset('allenai/c4', 'allenai--c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train')

	rng = random.Random(seed)

	trainloader = []
	for _ in range(nsamples):
		while True:
			i = rng.randint(0, len(traindata) - 1)
			trainenc = tokenizer(traindata[i]['text'], return_tensors='pt')
			if trainenc.input_ids.shape[1] >= seqlen:
				break

		i = rng.randint(0, trainenc.input_ids.shape[1] - seqlen - 1)
		inp = trainenc.input_ids[:, i:i + seqlen]
		trainloader.append(inp)

	return trainloader



#!/usr/bin/env python3
"""
Benchmarks the generation speed of a model.  While Benchmark.ipynb provides nice detailed performance data, it measures the kernels in isolation.
This script measures "real world" performance by running the whole model in generation mode.
It tests a grid of prompt lengths and generation lengths, and saves the timing results to `results.json`.
"""
import argparse
import itertools
import json
import os
import random
import time

import original_quant
import torch
import transformers
from gptq_triton import load_quant
from transformers import AutoTokenizer, LlamaConfig, LlamaForCausalLM


parser = argparse.ArgumentParser()
parser.add_argument('--model', type=str, help='Path to model, either a HuggingFace model or a quantized model')
parser.add_argument('--quant', action='store_true', help='Whether the model is quantized')
parser.add_argument('--cuda', type=str, help='Whether to use the old CUDA kernel and format; this must be set to the path to the CUDA quantized model, and --model must be set to a HF model')
parser.add_argument('--average', type=int, default=10, help='Number of times to run each test to get an average')


def main():
	args = parser.parse_args()

	if args.cuda:
		model = load_cuda_quant(args.model, args.cuda, 4, -1)
		model.eval()
		model.to('cuda')
	elif not args.quant:
		model = get_llama(args.model)
		model.eval()
		model.to('cuda')
	else:
		model = load_quant(args.model)
		model.eval()
		model.to('cuda')

	tokenizer = AutoTokenizer.from_pretrained(args.model, use_fast=False)

	prompt_lengths = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]
	max_lengths = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]

	lengths = set(itertools.product(prompt_lengths, max_lengths))

	# Remove lengths that we've already tested
	if os.path.exists('results.jsonl'):
		with open('results.jsonl', 'r') as f:
			for line in f:
				line = json.loads(line)
				key = (line['prompt_length'], line['max_length'])
				if key in lengths:
					lengths.remove(key)

	# Shuffle the lengths so that we don't always test in the same order and get caching effects
	lengths = list(lengths)
	random.shuffle(lengths)

	# TODO: For some reason the first run is always slow, so we run it once before the benchmark to warm things up
	encoded_prompt = tokenizer.encode("TODO", add_special_tokens=False, return_tensors='pt').to('cuda')
	_ = model.generate(
		input_ids=encoded_prompt,
		max_length=8,
		do_sample=True,
		num_return_sequences=1,
		suppress_tokens=[model.generation_config.eos_token_id],
	)

	# Run the remaining benchmarks
	with open('results.jsonl', 'a') as f:
		for prompt_length, max_length in lengths:
			print(f'Prompt length: {prompt_length}, max length: {max_length}')

			results = []

			for _ in range(args.average):
				# Generate a long random string
				# We do this every time to avoid caching effects
				prompt = ''.join(random.choice('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,;:!?') for _ in range(2048 * 10))

				# Encode and crop down
				encoded_prompt = tokenizer.encode(prompt, add_special_tokens=False, return_tensors='pt')
				encoded_prompt = encoded_prompt[:, :prompt_length]
				encoded_prompt = encoded_prompt.to('cuda')

				start_time = time.time()
				_ = model.generate(
					input_ids=encoded_prompt,
					max_length=max_length + prompt_length,
					do_sample=True,
					num_return_sequences=1,
					suppress_tokens=[model.generation_config.eos_token_id],  # This prevents the sampler from ending early; it must generate max_length tokens
				)
				end_time = time.time()

				gen_time = end_time - start_time
				speed = max_length / gen_time

				results.append((gen_time, speed))

			# Compute the average
			avg_time = sum(t for t, _ in results) / len(results)
			avg_speed = (max_length * len(results)) / sum(t for t, _ in results)

			print(f'Average generation time: {avg_time:.2f} seconds')
			print(f'Average generation speed: {avg_speed:.2f} tokens per second')
			print()

			f.write(json.dumps({
				'prompt_length': prompt_length,
				'max_length': max_length,
				'average_time': avg_time,
				'average_speed': avg_speed,
				'runs': results,
			}))
			f.write("\n")
			f.flush()


def get_llama(model: str):
	"""
	Load a pretrained Llama model
	"""
	def skip(*args, **kwargs):
		pass
	# NOTE: This is a nasty hack, but it speeds up model building by a huge amount
	old_inits = (torch.nn.init.kaiming_uniform_, torch.nn.init.uniform_, torch.nn.init.normal_)
	torch.nn.init.kaiming_uniform_ = skip
	torch.nn.init.uniform_ = skip
	torch.nn.init.normal_ = skip

	model = LlamaForCausalLM.from_pretrained(model, torch_dtype='auto')
	model.seqlen = 2048

	# Restore the old initializers
	torch.nn.init.kaiming_uniform_, torch.nn.init.uniform_, torch.nn.init.normal_ = old_inits

	return model


def load_cuda_quant(model, checkpoint, wbits, groupsize):
	"""
	Load a quantized model using the old CUDA kernel
	"""
	config = LlamaConfig.from_pretrained(model)
	def noop(*args, **kwargs):
		pass
	original_inits = (torch.nn.init.kaiming_uniform_, torch.nn.init.uniform_, torch.nn.init.normal_)
	torch.nn.init.kaiming_uniform_ = noop
	torch.nn.init.uniform_ = noop
	torch.nn.init.normal_ = noop

	torch.set_default_dtype(torch.half)
	original_init_weights = transformers.modeling_utils._init_weights
	transformers.modeling_utils._init_weights = False
	torch.set_default_dtype(torch.half)
	model = LlamaForCausalLM(config)
	torch.set_default_dtype(torch.float)

	transformers.modeling_utils._init_weights = original_init_weights
	torch.nn.init.kaiming_uniform_, torch.nn.init.uniform_, torch.nn.init.normal_ = original_inits

	model = model.eval()
	layers = original_quant.find_layers(model)
	for name in ['lm_head']:
		if name in layers:
			del layers[name]
	original_quant.make_quant(model, layers, wbits, groupsize, faster=False)

	del layers

	print('Loading model ...')
	if checkpoint.endswith('.safetensors'):
		from safetensors.torch import load_file as safe_load
		model.load_state_dict(safe_load(checkpoint))
	else:
		model.load_state_dict(torch.load(checkpoint))
	model.seqlen = 2048
	print('Done.')

	return model


if __name__ == '__main__':
	main()




VPS

Oct 12, 2023

Ignas R.

10min Read
25 Common Linux Bash Script Examples to Get You Started

Bash or Bourne-again shell is one of the most popular shells and command languages for Linux VPS enthusiasts. It was first released in 1989 and was used as the default shell for most Linux distributions ever since.

Bash scripting allows users and system administrators to automate processes and save hundreds of hours of manual work. It’s worth mentioning that Bash is also available for Windows and macOS.

This tutorial will introduce you to what bash scripting is. It features over twenty useful bash script examples to start your bash scripting journey.

    What Is Bash Scripting Used For
    25 Bash Scripts Examples
        1. Hello World
        2. Echo Command
        3. Sleep Command
        4. Wait Command
        5. Comments
        6. Get User Input
        7. Loops
        8. Create an Array
        9. Conditional Statements
        10. Functions
        11. Display String Length
        12. Extract String
        13. Find and Replace String
        14. Concatenate Strings
        15. Check if a Number is Even or Odd
        16. Generate Factorial of Number
        17. Create Directories
        18. Read Files
        19. Print Files With Line Count
        20. Delete Files
        21. Test if File Exists
        22. Check Inodes and Disk Usage
        23. Send Email Example
        24. Update Packages
        25. Show Server Information

What Is Bash Scripting Used For

Before we move on to the topic of bash scripting use cases, we need to elaborate on what bash and bash scripting are.

Bash is a command-line interface interpreter that runs in a text window where users can manage and execute shell commands. Bash – or shell scripting – on the other hand is the process of writing a set of commands to be executed on a Linux system. A file that includes such instructions is called a bash script.

To put it simply, the bash interpreter reads the bash script and executes the commands at the same time. For example, a Linux user can execute hundreds of commands with a single click instead of inputting them one by one. For this reason, bash scripting is the go-to choice for increasing productivity, setting up automation, and eliminating repetitive tasks.
25 Bash Scripts Examples

The following section will cover 25 of the most popular bash scripting examples, including variable manipulation and echoing out various values. We will also cover functions, arrays, loops, and much more.
1. Hello World

Hello World is the most simple bash script to start with. We will create a new variable called learningbash and print out the words Hello World. First, open a new shell script file with a text editor of your choice:

nano hello.sh

Paste the following lines into it:

#!/bin/bash
#Creates a new variable with a value of "Hello World"
learningbash="Hello World"
echo $learningbash

The command-line window showcasing the output of the first bash script – Hello World

The first line (/bin/bash) is used in every bash script. It instructs the operating system to use a bash interpreter as a command interpreter.
2. Echo Command

The echo bash command can be used to print out text as well as values of variables. In the following example, we will showcase how quotation marks affect the echo command. We will start by opening a new bash script file:

nano echo.sh

This simple bash script example will create a new variable and print it out while using different quotation marks.

#!/bin/bash
provider="Hostinger"
echo 'The best hosting provider is $provider'
echo "The best hosting provider is $provider"

The command-line window shows the echo command

As you can see, if the echo bash command is used with double quotation marks ““, then the script will print out the actual value of a variable. Otherwise, if the single quotation marks ‘‘ are used, it will print out only the name of a variable.
3. Sleep Command

Sleep command halts all currently running bash scripts and puts the system to sleep. Start by creating a new bash script file:

nano sleep.sh

Then, paste in the following simple script:

#!/bin/bash
sleep 10 && echo “I’ve been sleeping for 10 seconds, I want more” && sleep 10 && echo “I’m done sleeping, thanks!”

A bash script with the sleep command. The basic idea is that it puts the system to a halt for a set amount of time

The above example starts with a simple sleep bash command that will put your system to sleep for 10 seconds. After that, we combine the previously learned echo command with sleep – this way system will sleep for 10 seconds, then print out some words, sleep again, print out some words again and end its operation.
Pro Tip

A bash script can always be terminated by clicking CTRL + C without waiting for it to finish its operation.
4. Wait Command

wait is a built-in Linux command that waits for completion of running process. The wait command is used with a particular process id or job id.

Here’s how to create a wait bash script. Begin by creating a new bash file:

nano wait.sh

Paste in the following:

#!/bin/bash
wait 1234
echo “Done”

Important! If no job ID is provided, the wait command waits until all child background jobs are completed.
5. Comments

Users can easily add comments to their bash scripts with the # symbol. It is extra useful if you’ve got a lengthy script that needs explaining on some lines.

Begin by creating a new bash script:

nano comments.sh

Then paste in the following:

#!/bin/bash
# Define a variable named Hostinger
provider="Hostinger"
# Print out the following text
echo 'The best hosting provider is $provider'
# Print out the following text with $provider variable value
echo "The best hosting provider is $provider"

The command-line window showing single line comment functionality. It's worth noting that bash comments are not displayed with the script output.

Keep in mind that bash comments are only visible on a text editor.
6. Get User Input

To take input from users, we’ll use the read bash command. First, create a new bash shell file:

nano read.sh

Then, fill it with the script below:

#!/bin/bash
echo "What is your age?"
read age
echo "Wow, you look younger than $age years old"

In the above example, an age value was entered by the user. The output was then printed via the echo command.
7. Loops

A loop is an essential tool in various programming languages. To put it simply, a bash loop is a set of instructions that are repeated until a user-specified condition is reached. Start by creating a loop bash program:

nano whileloop.sh

Then paste in the following:

#!/bin/bash
n=0
while :
do
echo Countdown: $n
((n++))
done

This will work as a countdown to infinity until you press CTRL + C to stop the script.

Now that we’ve tested the while loop, we can move on to the for loop. Create a bash file for it:

nano forloop.sh

It should contain the script below:

#!/bin/bash
for (( n=2; n<=10; n++ ))
do
echo "$n seconds"
done

A bash script showcasing the "for" loop

The script prints out numbers from 2 to 10 while adding the seconds keyword to it.
8. Create an Array

A bash array is a data structure designed to store information in an indexed way. It is extra useful if users need to store and retrieve thousands of pieces of data fast. What makes bash arrays special is that unlike any other programming language, they can store different types of elements. For example, you can use a bash array to store both strings and numbers.

Create a new file in the current directory:

nano array.sh

Combine the freshly learned for loop with a new indexed array:

#!/bin/bash
# Create an indexed array
IndexedArray=(egg burger milk)
#Iterate over the array to get all the values
for i in "${IndexedArray[@]}";do echo "$i";done

A bash script to create and print out an array

The script iterates over the IndexedArray and prints out all the values.
9. Conditional Statements

The most popular and widely used conditional statement is if. Even though the if statement is easy to write and understand, it can be used in advanced shell scripts as well.

Begin with a new bash file:

nano if.sh

Paste the code below in it:

#!/bin/bash
salary=1000
expenses=800
#Check if salary and expenses are equal
if [ $salary == $expenses ];
then
    echo "Salary and expenses are equal"
#Check if salary and expenses are not equal
elif [ $salary != $expenses ];
then
    echo "Salary and expenses are not equal"
fi

This script creates two new variables and compares whether they are equal or not.
10. Functions

A bash function is a set of commands that can be reused numerous times throughout a bash script. Create a new file:

nano function.sh

Then, paste in the following code – it creates a simple Hello World function.

#!/bin/bash
hello () {
   echo 'Hello World!'
}
hello

11. Display String Length

There are a couple of ways of counting string length in bash. We’ll talk about the simplest. Create a file named stringlength.sh:

nano stringlength.sh

Fill it with the following:

#!/bin/bash
# Create a new string
mystring="lets count the length of this string"
i=${#mystring}
echo "Length: $i"

Here, the # operator is used to get the length of the string variable.
12. Extract String

If users need to remove unnecessary parts from strings, they can use the Bash string extraction tools. Start by creating a new bash script:

nano extractstring.sh

The following script has 4 values, 3 of them being strings. In our example, we will extract only the number value. This can be done via the cut command. First, we instruct the command that each variable is separated by a comma by using the -d flag. Then we ask the cut command to extract the 5th value.

#!/bin/bash
cut -d , -f 5 <<< "Website,Domain,DNS,SMTP,5005"

In another example, we have a string that is mixed with some numbers. We will use expr substr commands to extract only the Hostinger text value.

#!/bin/bash
expr substr "458449Hostinger4132" 7 9

13. Find and Replace String

Another useful bash script for strings is find and replace. Create a file named findreplace.sh:

nano findreplace.sh

Then paste in the following bash script:

#!/bin/bash
first="I drive a BMW and Volvo"
second="Audi"
echo "${first/BMW/"$second"}"

Find and replace script in bash

The find and replace functionality doesn’t require any special commands, it can all be done with string manipulation.
14. Concatenate Strings

Concatenation is the term used for appending one string to the end of another string. Start by creating concatenation.sh file.

nano concatenation.sh

The most simple example would be the following:

#!/bin/bash
firststring="The secret is..."
secondstring="Bash"
thirdstring="$firststring$secondstring"
echo "$thirdstring"

The above script will connect the values of firststring and secondstring variables creating a whole new thirdstring.

A more advanced example would look like this:

#!/bin/bash
firststring="The secret is..."
firststring+="Bash"
echo "$firststring"

The script uses the += operator to join the strings. With this method, you can concatenate strings with only one variable.
15. Check if a Number is Even or Odd

Odd and even numbers can be easily divided using the if statement and some simple math. Create a file named evenoddnumbers.sh:

nano evenoddnumbers.sh

The script uses the read command to read user input and divides it by 2. If the answer is 0, the number is even.

#!/bin/bash
read -p "Enter a number and I will check if its odd or even " mynumber
if [ $((mynumber%2)) -eq 0 ]
then
echo "Your number is even"
else
echo "Your number is odd."
fi

16. Generate Factorial of Number

The factorial of a number is the result of all positive descending integers. For example, the factorial of 5 would be 120:

5! = 5*4*3*2*1 = 120

Factorial scrips are very useful for users learning about recursion. Start by creating a .sh file executable:

factorial.sh

The following script will ask the user to enter a number they want to get the factorial of and use a for loop to calculate it.

#!/bin/bash
echo Enter the number you want to get factorial for
read mynumber
factorial=1
for ((i=1;i<=mynumber;i++))
do
factorial=$(($factorial*$i))
done
echo $factorial

The command-line window displaying shell script for getting factorial of a number
17. Create Directories

It is effortless to create directories in bash unless you need to create a lot of directories quickly. In the following example, we will use the bash script to create a set of directories with the same subdirectories in each.

First, create a file named directories.sh:

nano directories.sh

Then paste in the following code:

#!/bin/bash
mkdir -p {Math,English,Geography,Arts}/{notes,examresults,portfolio}

The script creates 4 main directories: Math, English, Geography, and Arts. The Notes, examresults, and portfolio subdirectories are also created inside each.

If you were to replace the / symbol in the middle with _, the script would look like this:

#!/bin/bash
mkdir -p {Math,English,Geography,Arts}_{notes,examresults,portfolio}

Here’s the output for it displaying a merge of the two directories:
A bash script to create a lot of directories quickly. The first bash case statement indicates which interpreter to use

#!/bin/bash
#Declare string S1
S1="Bash"
#Declare string S2
S2="Scripting"
if [ $S1 = $S2 ]; then
	echo "Both Strings are equal"
else
	echo "Strings are NOT equal"
fi


The following script will check to see if a file exists or not.

#!/bin/bash
file="./file"
if [ -e $file ]; then
	echo "File exists"
else
	echo "File does not exist"
fi

The result:

$ ./filetesting.sh
File does not exist
$ touch file
$ ./filetesting.sh
File exists

Similarly for example we can use while loop to check if file does not exist. This script will sleep until file does exist. Note bash negator ! which negates the -e option.

#!/bin/bash

while [ ! -e myfile ]; do
# Sleep until file does exists/is created
sleep 1
done


Bash Select

The select command allows us to prompt the user to make a selection.

#!/bin/bash

PS3='Choose one word: '

# bash select
select word in "linux" "bash" "scripting" "tutorial"
do
  echo "The word you have selected is: $word"
# Break, otherwise endless loop
  break
done

exit 0

The result:

$ ./select.sh
1) linux
2) bash
3) scripting
4) tutorial
Choose one word: 2
The word you have selected is: bash

Case statement conditional

The case statement makes it easy to have many different possibilities, whereas an if statement can get lengthy very quickly if you have more than a few possibilities to account for.

#!/bin/bash
echo "What is your preferred programming / scripting language"
echo "1) bash"
echo "2) perl"
echo "3) phyton"
echo "4) c++"
echo "5) I do not know !"
read case;
#simple case bash structure
# note in this case $case is variable and does not have to
# be named case this is just an example
case $case in
    1) echo "You selected bash";;
    2) echo "You selected perl";;
    3) echo "You selected phyton";;
    4) echo "You selected c++";;
    5) exit
esac

The result:

$ ./case.sh
What is your preferred programming / scripting language
1) bash
2) perl
3) phyton
4) c++
5) I do not know !
3
You selected phyton

Bash quotes and quotations

Quotations and quotes are important part of bash and bash scripting. Here are some bash quotes and quotations basics.
Escaping Meta characters

Before we start with quotes and quotations we should know something about escaping meta characters. Escaping will suppress a special meaning of meta characters and therefore meta characters will be read by bash literally. To do this we need to use backslash \ character. Example:

#!/bin/bash

#Declare bash string variable
BASH_VAR="Bash Script"

# echo variable BASH_VAR
echo $BASH_VAR

#when meta character such us "$" is escaped with "\" it will be read literally
echo \$BASH_VAR

# backslash has also special meaning and it can be suppressed with yet another "\"
echo "\\"

Here’s what it looks like when we execute the script:

$ ./escape_meta.sh
Bash Script
$BASH_VAR
\

Single quotes

Single quotes in bash will suppress special meaning of every meta characters. Therefore meta characters will be read literally. It is not possible to use another single quote within two single quotes not even if the single quote is escaped by backslash.

#!/bin/bash

# Declare bash string variable
BASH_VAR="Bash Script"

# echo variable BASH_VAR
echo $BASH_VAR

# meta characters special meaning in bash is suppressed when  using single quotes
echo '$BASH_VAR  "$BASH_VAR"'

The result:

$ ./single_quotes.sh
Bash Script
$BASH_VAR "$BASH_VAR"

Double quotes

Double quotes in bash will suppress special meaning of every meta characters except $, \ and `. Any other meta characters will be read literally. It is also possible to use single quote within double quotes. If we need to use double quotes within double quotes bash can read them literally when escaping them with \. Example:

#!/bin/bash

#Declare bash string variable
BASH_VAR="Bash Script"

# echo variable BASH_VAR
echo $BASH_VAR

# meta characters and its special meaning in bash is
# suppressed when using double quotes except "$", "\" and "`"

echo "It's $BASH_VAR  and \"$BASH_VAR\" using backticks: `date`"

The result:

$ ./double_quotes.sh
Bash Script
It's Bash Script and "Bash Script" using backticks: Thu 10 Feb 2022 10:24:15 PM EST

Bash quoting with ANSI-C style

There is also another type of quoting and that is ANSI-C. In this type of quoting characters escaped with \ will gain special meaning according to the ANSI-C standard.
\a 	alert (bell) 	\b 	backspace
\e 	an escape character 	\f 	form feed
\n 	newline 	\r 	carriage return
\t 	horizontal tab 	\v 	vertical tab
\\ 	backslash 	\` 	single quote
\nnn 	octal value of characters ( see [http://www.asciitable.com/ ASCII table] ) 	\xnn 	hexadecimal value of characters ( see [http://www.asciitable.com/ ASCII table] )

